def update_committee_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014): #print "Updating %s" % (committee_id) c = None try: c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from) except Committee.MultipleObjectsReturned: print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from) return None except Committee.DoesNotExist: print "Missing committee with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from) return None committee_overlay = None try: committee_overlay = Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from) except Committee_Overlay.DoesNotExist: # This shouldn't happen return None ctype = c.cmte_tp is_hybrid = False is_noncommittee = False is_superpac = False if ctype: if ctype.upper() in ['O', 'U']: is_superpac = True if ctype.upper() in ['V', 'W']: is_hybrid = True if ctype.upper() in ['I']: is_noncommittee = True party = c.cmte_pty_affiliation if party: party = get_party_from_pty(party) committee_overlay.cycle = cycle_to_copy_to committee_overlay.name = c.cmte_name committee_overlay.slug = slugify(c.cmte_name) committee_overlay.party = party committee_overlay.treasurer = c.tres_nm committee_overlay.street_1 = c.cmte_st1 committee_overlay.street_2 = c.cmte_st2 committee_overlay.city = c.cmte_city committee_overlay.state = c.cmte_st committee_overlay.connected_org_name = c.connected_org_nm committee_overlay.filing_frequency = c.cmte_filing_freq committee_overlay.candidate_id = c.cand_id committee_overlay.is_superpac = is_superpac committee_overlay.is_hybrid = is_hybrid committee_overlay.is_noncommittee = is_noncommittee committee_overlay.designation = c.cmte_dsgn committee_overlay.ctype = ctype committee_overlay.save()
def get_senate_challengers(): # then get the candidates who are declared. While we set the incumbents' election year as 2014--overriding whatever they have in their FEC filings, we don't do that here. allSenateCandidates = Candidate.objects.filter(cycle=cycle, cand_office='S', cand_election_year__gte=2013) for thiscandidate in allSenateCandidates: fec_id = thiscandidate.cand_id try: senate_exclusion_hash[fec_id] continue except KeyError: pass #print "handling %s" % (thiscandidate.cand_name) # do we already have this candidate entered ? try: entered_candidate = Candidate_Overlay.objects.get(election_year__gte=2013, fec_id=fec_id) #print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici) except Candidate_Overlay.DoesNotExist: # don't enter if they say they're an incumbent. Problaby an error. if thiscandidate.cand_ici == 'I': print "!! Not-yet-entered Senate candidate %s %s %s claims incumbency. Perhaps they've retired?" % (thiscandidate.cand_name, thiscandidate.cand_election_year, thiscandidate.cand_office_st) else: # It's a new candidate. Enter it. state = thiscandidate.cand_office_st term_class = get_term_class_from_election_year(thiscandidate.cand_election_year) if thiscandidate.cand_election_year < 2013: print "!! Disregarding old candidate %s %s %s" % (thiscandidate.cand_name, thiscandidate.cand_election_year, state) election_year = str(thiscandidate.cand_election_year) try: this_district = District.objects.get(election_year=election_year, state=state, office='S') # Only make it an open seat if we've already marked the district as open. cand_ici = 'C' if this_district.open_seat: cand_ici='O' Candidate_Overlay.objects.create( district=this_district, cycle=cycle, fec_id=fec_id, name=thiscandidate.cand_name, slug = slugify(thiscandidate.cand_name), pty=thiscandidate.cand_pty_affiliation, party = get_party_from_pty(thiscandidate.cand_pty_affiliation), #pcc=thiscandidate.cand_pcc, term_class=term_class, election_year=thiscandidate.cand_election_year, state=thiscandidate.cand_office_st, office='S', cand_ici=cand_ici, candidate_status='D' ) except District.DoesNotExist: print "!! Invalid senate district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state) continue
def get_senate_incumbents(): # Get the incumbents for member in senate_crosswalk: bio = member['bioguide'] fec_id = member['fec_id'] if not fec_id: # ignore continue #print "handling %s %s" % (bio, fec_id) # term class isn't kept in the fec records... thisterm = Term.objects.get(start__lte=cycle_end, end__gte=cycle_start, term_type='sen', legislator__bioguide=bio) thiscandidate = Candidate.objects.get(cand_id=fec_id, cycle=cycle) office = thiscandidate.cand_office term_class = thisterm.term_class state = thiscandidate.cand_office_st #print "processing %s: %s %s %s" % (thiscandidate.cand_name, office, state, term_class) election_year = get_election_year_from_term_class(term_class) senate_cycle = str(election_year) this_district = District.objects.get(cycle=senate_cycle,state=state, office='S', term_class=term_class) this_overlay, created = Candidate_Overlay.objects.get_or_create(fec_id=fec_id, cycle=senate_cycle, district=this_district) this_overlay.name = thiscandidate.cand_name this_overlay.slug = slugify(thiscandidate.cand_name) this_overlay.candidate = thiscandidate this_overlay.office='S' this_overlay.state = thiscandidate.cand_office_st this_overlay.pty = thiscandidate.cand_pty_affiliation this_overlay.party = get_party_from_pty(thiscandidate.cand_pty_affiliation) this_overlay.pcc = thiscandidate.cand_pcc this_overlay.cand_ici = 'I' this_overlay.election_year = election_year this_overlay.term_class=term_class this_overlay.save() try: senate_casualty_hash[fec_id] this_overlay.not_seeking_reelection = True this_overlay.save() this_district.open_seat = True this_district.save() except KeyError: pass
def get_house_incumbents(): # Get the incumbents for member in house_crosswalk: bio = member['bioguide'] fec_id = member['fec_id'] thiscandidate = Candidate.objects.get(cand_id=fec_id, cycle=cycle) #office = thiscandidate.cand_office office = 'H' district = None district = thiscandidate.cand_office_district.zfill(2) state = thiscandidate.cand_office_st #print "processing %s: %s %s %s" % (thiscandidate.cand_name, office, state, district) this_district = District.objects.get(cycle=cycle,state=state, office='H', office_district=district) this_overlay, created = Candidate_Overlay.objects.get_or_create(fec_id=fec_id, cycle=cycle, district=this_district) this_overlay.name = thiscandidate.cand_name this_overlay.slug = slugify(thiscandidate.cand_name) this_overlay.pty = thiscandidate.cand_pty_affiliation this_overlay.party = get_party_from_pty(thiscandidate.cand_pty_affiliation) this_overlay.pcc = thiscandidate.cand_pcc this_overlay.cand_ici = 'I' this_overlay.office = office this_overlay.election_year = 2014 this_overlay.save() try: house_casualty_hash[fec_id] this_overlay.not_seeking_reelection = True this_overlay.save() this_district.open_seat = True this_district.save() except KeyError: pass
def make_candidate_overlay_from_masterfile(candidate_id, cycle_to_copy_from=2014, election_year=2014, cycle_to_copy_to=2014, verify_does_not_exist=True, display_candidate=False): ## Returns overlay if created, None if not. if verify_does_not_exist: try: # If there's already a candidate overlay, don't do this. entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id) #print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici) return None except Candidate_Overlay.DoesNotExist: pass thiscandidate = None try: thiscandidate = Candidate.objects.get(cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year) except Candidate.DoesNotExist: print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % (candidate_id, election_year, cycle_to_copy_from) return None state = thiscandidate.cand_office_st term_class = None if thiscandidate.cand_office == 'S': term_class = get_term_class_from_election_year(thiscandidate.cand_election_year) this_district = None try: if thiscandidate.cand_office == 'S': this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class) elif thiscandidate.cand_office == 'H': this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, office_district=thiscandidate.cand_office_district) elif thiscandidate.cand_office == 'P': # there's no district, so pass # do we need a presidential placeholder ? pass except District.DoesNotExist: print "!! Invalid district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state) # If we can't find a district, override the display setting--just don't display it. display_candidate = False co = Candidate_Overlay.objects.create( district=this_district, office_district=thiscandidate.cand_office_district, cycle=cycle_to_copy_to, fec_id=candidate_id, name=thiscandidate.cand_name, slug = slugify(thiscandidate.cand_name), pty=thiscandidate.cand_pty_affiliation, party = get_party_from_pty(thiscandidate.cand_pty_affiliation), pcc=thiscandidate.cand_pcc, term_class=term_class, election_year=thiscandidate.cand_election_year, curated_election_year=thiscandidate.cand_election_year, state=thiscandidate.cand_office_st, office=thiscandidate.cand_office, cand_ici=thiscandidate.cand_ici, candidate_status='D', display = display_candidate, ) return co
def make_committee_overlay_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014, verify_does_not_exist=True): c = None try: c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from) except Committee.MultipleObjectsReturned: print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from) return None if verify_does_not_exist: try: Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from) # if it exists, update it with the current information update_committee_from_masterfile(committee_id, cycle_to_copy_from=2014, cycle_to_copy_to=2014) return None except Committee_Overlay.DoesNotExist: pass ctype = c.cmte_tp is_hybrid = False is_noncommittee = False is_superpac = False is_paper_filer = False if ctype: if ctype.upper() in ['O', 'U']: is_superpac = True if ctype.upper() in ['V', 'W']: is_hybrid = True if ctype.upper() in ['I']: is_noncommittee = True if ctype.upper() in ['S']: is_paper_filer = True party = c.cmte_pty_affiliation if party: party = get_party_from_pty(party) cm = Committee_Overlay.objects.create( cycle = cycle_to_copy_to, name = c.cmte_name, fec_id = c.cmte_id, slug = slugify(c.cmte_name), party = party, treasurer = c.tres_nm, street_1 = c.cmte_st1, street_2 = c.cmte_st2, city = c.cmte_city, state = c.cmte_st, connected_org_name = c.connected_org_nm, filing_frequency = c.cmte_filing_freq, candidate_id = c.cand_id, is_superpac = is_superpac, is_hybrid = is_hybrid, is_noncommittee = is_noncommittee, designation = c.cmte_dsgn, ctype = ctype, is_dirty=True, is_paper_filer=is_paper_filer ) return cm
def handle_filing(this_filing): try: co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id, cycle=this_filing.cycle) this_filing.committee_designation = co.designation this_filing.committee_name = co.name this_filing.committee_type = co.ctype this_filing.committee_slug = co.slug this_filing.party = co.party # mark that the committee is dirty co.is_dirty = True co.save() except Committee_Overlay.DoesNotExist: try: ## remember that ftpdata committees have cycles as ints, not strings. Not ideal. if not this_filing.cycle: this_filing.cycle = CURRENT_CYCLE co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=int(this_filing.cycle)) this_filing.committee_designation = co.cmte_dsgn this_filing.committee_type = co.cmte_tp this_filing.committee_name = co.cmte_name this_filing.party = get_party_from_pty(co.cmte_pty_affiliation) except Committee.DoesNotExist: pass header_data = this_filing.header_data form_type = this_filing.form_type parsed_data = { 'coh_start': None, 'coh_end': None, 'new_loans': None, 'tot_raised': None, 'tot_spent': None } if form_type in ['F3A', 'F3N', 'F3T', 'F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']: parsed_data = process_f3_header(header_data) print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data[ 'coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data[ 'new_loans'] else 0 this_filing.new_filing_details_set = True elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']: parsed_data = process_f3x_header(header_data) print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data[ 'coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data[ 'new_loans'] else 0 this_filing.tot_coordinated = parsed_data[ 'tot_coordinated'] if parsed_data['tot_coordinated'] else 0 this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data[ 'tot_ies'] else 0 this_filing.new_filing_details_set = True elif form_type in ['F5', 'F5A', 'F5N']: parsed_data = process_f5_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 # total spending is total ies this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] this_filing.coverage_to_date = parsed_data['coverage_to_date'] try: this_filing.is_f5_quarterly = header_data['report_code'] in [ 'Q1', 'Q2', 'Q3', 'Q4', 'YE' ] except KeyError: # this is probably a problem. pass this_filing.new_filing_details_set = True elif form_type in ['F7', 'F7A', 'F7N']: parsed_data = process_f7_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True elif form_type in ['F9', 'F9A', 'F9N']: parsed_data = process_f9_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True elif form_type in ['F13', 'F13A', 'F13N']: parsed_data = process_f13_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True else: # Nothing to be done, but mark this step as done. this_filing.new_filing_details_set = True this_filing.save()
def make_candidate_overlay_from_masterfile( candidate_id, cycle_to_copy_from=2016, election_year=2016, cycle_to_copy_to=2016, verify_does_not_exist=True, display_candidate=False, ): if candidate_id == "P20003851": return None ## Returns overlay if created, None if not. if verify_does_not_exist: try: # If there's already a candidate overlay, don't do this. entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id) # print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici) return None except Candidate_Overlay.DoesNotExist: pass thiscandidate = None try: thiscandidate = Candidate.objects.get( cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year ) except Candidate.DoesNotExist: print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % ( candidate_id, election_year, cycle_to_copy_from, ) return None state = thiscandidate.cand_office_st term_class = None if thiscandidate.cand_office == "S": term_class = get_term_class_from_election_year(thiscandidate.cand_election_year) this_district = None try: if thiscandidate.cand_office == "S": this_district = District.objects.get( election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class ) elif thiscandidate.cand_office == "H": this_district = District.objects.get( election_year=election_year, state=state, office=thiscandidate.cand_office, office_district=thiscandidate.cand_office_district, ) elif thiscandidate.cand_office == "P": # there's a single presidential district per cycle, it needs to be created manually this_district = District.objects.get(election_year=election_year, office=thiscandidate.cand_office) except District.DoesNotExist: print "!! Invalid %s district for %s term_class=%s district=%s election_year=%s state=%s" % ( thiscandidate.cand_office, thiscandidate.cand_name, term_class, thiscandidate.cand_office_district, thiscandidate.cand_election_year, state, ) # If we can't find a district, override the display setting--just don't display it. display_candidate = False co = Candidate_Overlay.objects.create( district=this_district, office_district=thiscandidate.cand_office_district, cycle=cycle_to_copy_to, fec_id=candidate_id, name=thiscandidate.cand_name, slug=slugify(thiscandidate.cand_name), pty=thiscandidate.cand_pty_affiliation, party=get_party_from_pty(thiscandidate.cand_pty_affiliation), pcc=thiscandidate.cand_pcc, term_class=term_class, election_year=thiscandidate.cand_election_year, curated_election_year=thiscandidate.cand_election_year, state=thiscandidate.cand_office_st, office=thiscandidate.cand_office, cand_ici=thiscandidate.cand_ici, candidate_status="D", display=display_candidate, ) return co
def make_committee_overlay_from_masterfile( committee_id, cycle_to_copy_from=2016, cycle_to_copy_to=2016, verify_does_not_exist=True ): if committee_id == "C00507947": return None c = None try: c = Committee.objects.get(cmte_id=committee_id, cycle=cycle_to_copy_from) except Committee.MultipleObjectsReturned: print "Multiple committees found with id=%s cycle=%s!" % (committee_id, cycle_to_copy_from) return None if verify_does_not_exist: try: Committee_Overlay.objects.get(fec_id=committee_id, cycle=cycle_to_copy_from) # if it exists, update it with the current information update_committee_from_masterfile(committee_id, cycle_to_copy_from=2016, cycle_to_copy_to=2016) return None except Committee_Overlay.DoesNotExist: pass ctype = c.cmte_tp is_hybrid = False is_noncommittee = False is_superpac = False is_paper_filer = False if ctype: if ctype.upper() in ["O", "U"]: is_superpac = True if ctype.upper() in ["V", "W"]: is_hybrid = True if ctype.upper() in ["I"]: is_noncommittee = True if ctype.upper() in ["S"]: is_paper_filer = True party = c.cmte_pty_affiliation if party: party = get_party_from_pty(party) cm = Committee_Overlay.objects.create( cycle=cycle_to_copy_to, name=c.cmte_name, fec_id=c.cmte_id, slug=slugify(c.cmte_name), party=party, treasurer=c.tres_nm, street_1=c.cmte_st1, street_2=c.cmte_st2, city=c.cmte_city, state=c.cmte_st, connected_org_name=c.connected_org_nm, filing_frequency=c.cmte_filing_freq, candidate_id=c.cand_id, is_superpac=is_superpac, is_hybrid=is_hybrid, is_noncommittee=is_noncommittee, designation=c.cmte_dsgn, ctype=ctype, is_dirty=True, is_paper_filer=is_paper_filer, ) return cm
def handle_filing(this_filing): try: co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id) this_filing.committee_designation = co.designation this_filing.committee_type = co.ctype this_filing.committee_slug = co.slug this_filing.party = co.party # mark that the committee is dirty co.is_dirty = True co.save() except Committee_Overlay.DoesNotExist: try: co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=2014) this_filing.committee_designation = co.cmte_dsgn this_filing.committee_type = co.cmte_tp this_filing.party = get_party_from_pty(co.cmte_pty_affiliation) except Committee.DoesNotExist: pass header_data = this_filing.header_data form_type = this_filing.form_type parsed_data = { 'coh_start': None, 'coh_end': None, 'new_loans': None, 'tot_raised': None, 'tot_spent': None } if form_type in ['F3A', 'F3N', 'F3T', 'F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']: parsed_data = process_f3_header(header_data) #print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data[ 'coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data[ 'new_loans'] else 0 elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']: parsed_data = process_f3x_header(header_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data[ 'coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data[ 'new_loans'] else 0 this_filing.tot_coordinated = parsed_data[ 'tot_coordinated'] if parsed_data['tot_coordinated'] else 0 this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data[ 'tot_ies'] else 0 elif form_type in ['F5', 'F5A', 'F5N']: parsed_data = process_f5_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 # total spending is total ies this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] this_filing.coverage_to_date = parsed_data['coverage_to_date'] this_filing.is_f5_quarterly = header_data['report_code'] in [ 'Q1', 'Q2', 'Q3', 'Q4', 'YE' ] elif form_type in ['F7', 'F7A', 'F7N']: parsed_data = process_f7_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None elif form_type in ['F9', 'F9A', 'F9N']: parsed_data = process_f9_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data[ 'tot_spent'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None elif form_type in ['F13', 'F13A', 'F13N']: parsed_data = process_f13_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data[ 'tot_raised'] else 0 this_filing.coverage_from_date = parsed_data[ 'coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data[ 'coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.save()
def handle_filing(this_filing): try: co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id) this_filing.committee_designation = co.designation this_filing.committee_type = co.ctype this_filing.committee_slug = co.slug this_filing.party = co.party # mark that the committee is dirty co.is_dirty=True co.save() except Committee_Overlay.DoesNotExist: try: co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=2014) this_filing.committee_designation = co.cmte_dsgn this_filing.committee_type = co.cmte_tp this_filing.party = get_party_from_pty(co.cmte_pty_affiliation) except Committee.DoesNotExist: pass header_data = this_filing.header_data form_type = this_filing.form_type parsed_data = {'coh_start':None, 'coh_end':None, 'new_loans':None,'tot_raised':None,'tot_spent':None} if form_type in ['F3A', 'F3N', 'F3T','F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']: parsed_data = process_f3_header(header_data) #print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0 elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']: parsed_data = process_f3x_header(header_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0 this_filing.tot_coordinated = parsed_data['tot_coordinated'] if parsed_data['tot_coordinated'] else 0 this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data['tot_ies'] else 0 elif form_type in ['F5', 'F5A', 'F5N']: parsed_data = process_f5_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 # total spending is total ies this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] this_filing.coverage_to_date = parsed_data['coverage_to_date'] this_filing.is_f5_quarterly = header_data['report_code'] in ['Q1', 'Q2', 'Q3', 'Q4', 'YE'] elif form_type in ['F7', 'F7A', 'F7N']: parsed_data = process_f7_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None elif form_type in ['F9', 'F9A', 'F9N']: parsed_data = process_f9_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None elif form_type in ['F13', 'F13A', 'F13N']: parsed_data = process_f13_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.save()
def make_candidate_overlay_from_masterfile(candidate_id, cycle_to_copy_from=2014, election_year=2014, cycle_to_copy_to=2014, verify_does_not_exist=True, display_candidate=False): if candidate_id == 'P20003851': return None ## Returns overlay if created, None if not. if verify_does_not_exist: try: # If there's already a candidate overlay, don't do this. entered_candidate = Candidate_Overlay.objects.get(cycle=cycle_to_copy_to, fec_id=candidate_id) #print "Found candidate %s status %s" % (entered_candidate.name, entered_candidate.cand_ici) return None except Candidate_Overlay.DoesNotExist: pass thiscandidate = None try: thiscandidate = Candidate.objects.get(cycle=cycle_to_copy_from, cand_id=candidate_id, cand_election_year=election_year) except Candidate.DoesNotExist: print "Couldn't find candidate in masterfile id=%s election_year=%s cycle=%s" % (candidate_id, election_year, cycle_to_copy_from) return None state = thiscandidate.cand_office_st term_class = None if thiscandidate.cand_office == 'S': term_class = get_term_class_from_election_year(thiscandidate.cand_election_year) this_district = None try: if thiscandidate.cand_office == 'S': this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, term_class=term_class) elif thiscandidate.cand_office == 'H': this_district = District.objects.get(election_year=election_year, state=state, office=thiscandidate.cand_office, office_district=thiscandidate.cand_office_district) elif thiscandidate.cand_office == 'P': # there's no district, so pass # do we need a presidential placeholder ? pass except District.DoesNotExist: print "!! Invalid district for %s %s %s %s" % (thiscandidate.cand_name, term_class, thiscandidate.cand_election_year, state) # If we can't find a district, override the display setting--just don't display it. display_candidate = False co = Candidate_Overlay.objects.create( district=this_district, office_district=thiscandidate.cand_office_district, cycle=cycle_to_copy_to, fec_id=candidate_id, name=thiscandidate.cand_name, slug = slugify(thiscandidate.cand_name), pty=thiscandidate.cand_pty_affiliation, party = get_party_from_pty(thiscandidate.cand_pty_affiliation), pcc=thiscandidate.cand_pcc, term_class=term_class, election_year=thiscandidate.cand_election_year, curated_election_year=thiscandidate.cand_election_year, state=thiscandidate.cand_office_st, office=thiscandidate.cand_office, cand_ici=thiscandidate.cand_ici, candidate_status='D', display = display_candidate, ) return co
def handle_filing(this_filing): try: co = Committee_Overlay.objects.get(fec_id=this_filing.fec_id, cycle=this_filing.cycle) this_filing.committee_designation = co.designation this_filing.committee_name = co.name this_filing.committee_type = co.ctype this_filing.committee_slug = co.slug this_filing.party = co.party # mark that the committee is dirty co.is_dirty=True co.save() except Committee_Overlay.DoesNotExist: try: ## remember that ftpdata committees have cycles as ints, not strings. Not ideal. if not this_filing.cycle: this_filing.cycle = CURRENT_CYCLE co = Committee.objects.get(cmte_id=this_filing.fec_id, cycle=int(this_filing.cycle)) this_filing.committee_designation = co.cmte_dsgn this_filing.committee_type = co.cmte_tp this_filing.committee_name = co.cmte_name this_filing.party = get_party_from_pty(co.cmte_pty_affiliation) except Committee.DoesNotExist: pass header_data = this_filing.header_data form_type = this_filing.form_type parsed_data = {'coh_start':None, 'coh_end':None, 'new_loans':None,'tot_raised':None,'tot_spent':None} if form_type in ['F3A', 'F3N', 'F3T','F3PA', 'F3PN', 'F3PT', 'F3', 'F3P']: parsed_data = process_f3_header(header_data) print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0 this_filing.new_filing_details_set = True elif form_type in ['F3X', 'F3XA', 'F3XN', 'F3XT']: parsed_data = process_f3x_header(header_data) print "got data %s" % (parsed_data) this_filing.coh_end = parsed_data['coh_end'] if parsed_data['coh_end'] else 0 this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.new_loans = parsed_data['new_loans'] if parsed_data['new_loans'] else 0 this_filing.tot_coordinated = parsed_data['tot_coordinated'] if parsed_data['tot_coordinated'] else 0 this_filing.tot_ies = parsed_data['tot_ies'] if parsed_data['tot_ies'] else 0 this_filing.new_filing_details_set = True elif form_type in ['F5', 'F5A', 'F5N']: parsed_data = process_f5_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 # total spending is total ies this_filing.tot_ies = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] this_filing.coverage_to_date = parsed_data['coverage_to_date'] try: this_filing.is_f5_quarterly = header_data['report_code'] in ['Q1', 'Q2', 'Q3', 'Q4', 'YE'] except KeyError: # this is probably a problem. pass this_filing.new_filing_details_set = True elif form_type in ['F7', 'F7A', 'F7N']: parsed_data = process_f7_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True elif form_type in ['F9', 'F9A', 'F9N']: parsed_data = process_f9_header(header_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.tot_spent = parsed_data['tot_spent'] if parsed_data['tot_spent'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True elif form_type in ['F13', 'F13A', 'F13N']: parsed_data = process_f13_header(header_data) #print "got data %s" % (parsed_data) this_filing.tot_raised = parsed_data['tot_raised'] if parsed_data['tot_raised'] else 0 this_filing.coverage_from_date = parsed_data['coverage_from_date'] if parsed_data['coverage_from_date'] else None this_filing.coverage_to_date = parsed_data['coverage_to_date'] if parsed_data['coverage_to_date'] else None this_filing.new_filing_details_set = True else: # Nothing to be done, but mark this step as done. this_filing.new_filing_details_set = True this_filing.save()