def summarize_committee_periodic_electronic(committee_id, cycle, force_update=True): # it's a pain, but we need the committee name in this model. committee_name = "" try: this_committee = Committee.objects.get(cmte_id=committee_id, cycle=cycle) committee_name = this_committee.cmte_name except Committee.DoesNotExist: print "Missing committee name" pass except Committee.MultipleObjectsReturned: print "multiple committees!! id=%s" % (committee_id) pass this_cycle_calendar = cycle_calendar[int(cycle)] this_cycle_start = this_cycle_calendar['start'] this_cycle_end = this_cycle_calendar['end'] relevant_filings = new_filing.objects.filter( fec_id=committee_id, is_superceded=False, coverage_from_date__gte=this_cycle_start, coverage_to_date__lte=this_cycle_end, form_type__in=[ 'F3P', 'F3PN', 'F3PA', 'F3PT', 'F3', 'F3A', 'F3N', 'F3T', 'F3X', 'F3XA', 'F3XN', 'F3XT' ]).order_by('coverage_from_date') #print "processing %s" % committee_id if not relevant_filings: #print "No filings found for %s" % (committee_id) return None # check gaps last_end_date = None for i, this_filing in enumerate(relevant_filings): #print i, this_filing.coverage_from_date, this_filing.coverage_through_date if i == 0: if this_filing.coverage_from_date - this_cycle_start > one_day: pass #print "Missing coverage from start of cycle!!" # set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id) if i > 0: difference = this_filing.coverage_from_date - last_end_date if difference > one_day: pass #print "gap found!" # set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id) #print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date) last_end_date = this_filing.coverage_to_date form = this_filing.form_type header_data = this_filing.header_data cts_dict = map_summary_form_to_dict(form, header_data) #print "Form = %s cts_dict = %s" % (form, cts_dict) tot_contribs = string_to_float( cts_dict['tot_ite_contrib']) + string_to_float( cts_dict['tot_non_ite_contrib']) cts_dict['filing_number'] = this_filing.filing_number cts_dict['coverage_through_date'] = this_filing.coverage_to_date cts_dict['coverage_from_date'] = this_filing.coverage_from_date cts_dict['data_source'] = 'electronic' cts_dict['com_id'] = committee_id cts_dict['tot_contrib'] = tot_contribs cts_dict['com_name'] = committee_name for i in cts_dict: if cts_dict[i] == '': cts_dict[i] = None try: this_summary = Committee_Time_Summary.objects.get( com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date) if force_update: this_summary.filing_number = this_filing.filing_number this_summary.tot_receipts = cts_dict.get('tot_receipts') this_summary.tot_ite_contrib = cts_dict.get('tot_ite_contrib') this_summary.tot_non_ite_contrib = cts_dict.get( 'tot_non_ite_contrib') this_summary.tot_contrib = cts_dict.get('tot_contrib') this_summary.com_name = cts_dict.get('com_name') this_summary.tot_disburse = cts_dict.get('tot_disburse') this_summary.new_loans = cts_dict.get('new_loans') this_summary.outstanding_loans = cts_dict.get( 'outstanding_loans') this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad') this_summary.coo_exp_par = cts_dict.get('coo_exp_par') this_summary.cash_on_hand_end = cts_dict.get( 'cash_on_hand_end') this_summary.data_source = cts_dict.get('data_source') this_summary.save() except Committee_Time_Summary.DoesNotExist: cts = Committee_Time_Summary(**cts_dict) print "creating committee summary %s" % cts_dict cts.save()
def summarize_committee_periodic_electronic(committee_id, force_update=True): # it's a pain, but we need the committee name in this model. committee_name = "" try: this_committee = Committee.objects.get(cmte_id=committee_id, cycle=CYCLE) committee_name = this_committee.cmte_name except Committee.DoesNotExist: print "Missing committee name" pass except Committee.MultipleObjectsReturned: print "multiple committees!! id=%s" % (committee_id) pass relevant_filings = new_filing.objects.filter(fec_id=committee_id, is_superceded=False, coverage_from_date__gte=date(2013,1,1), form_type__in=['F3P', 'F3PN', 'F3PA', 'F3PT', 'F3', 'F3A', 'F3N', 'F3T', 'F3X', 'F3XA', 'F3XN', 'F3XT']).order_by('coverage_from_date') #print "processing %s" % committee_id if not relevant_filings: #print "No filings found for %s" % (committee_id) return None # check gaps last_end_date = None for i, this_filing in enumerate(relevant_filings): #print i, this_filing.coverage_from_date, this_filing.coverage_through_date if i==0: if this_filing.coverage_from_date - this_cycle_start > one_day: #print "Missing coverage from start of cycle!!" set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id) if i>0: difference = this_filing.coverage_from_date - last_end_date if difference > one_day: #print "gap found!" set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id) #print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date) last_end_date = this_filing.coverage_to_date form = this_filing.form_type header_data = this_filing.header_data cts_dict = map_summary_form_to_dict(form, header_data) #print "Form = %s cts_dict = %s" % (form, cts_dict) tot_contribs = string_to_float(cts_dict['tot_ite_contrib']) + string_to_float(cts_dict['tot_non_ite_contrib']) cts_dict['filing_number'] = this_filing.filing_number cts_dict['coverage_through_date'] = this_filing.coverage_to_date cts_dict['coverage_from_date'] = this_filing.coverage_from_date cts_dict['data_source'] = 'electronic' cts_dict['com_id'] = committee_id cts_dict['tot_contrib'] = tot_contribs cts_dict['com_name'] = committee_name for i in cts_dict: if cts_dict[i] == '': cts_dict[i] = None try: this_summary = Committee_Time_Summary.objects.get(com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date) if force_update: this_summary.filing_number = this_filing.filing_number this_summary.tot_receipts = cts_dict.get('tot_receipts') this_summary.tot_ite_contrib = cts_dict.get('tot_ite_contrib') this_summary.tot_non_ite_contrib = cts_dict.get('tot_non_ite_contrib') this_summary.tot_contrib = cts_dict.get('tot_contrib') this_summary.com_name = cts_dict.get('com_name') this_summary.tot_disburse = cts_dict.get('tot_disburse') this_summary.new_loans = cts_dict.get('new_loans') this_summary.outstanding_loans = cts_dict.get('outstanding_loans') this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad') this_summary.coo_exp_par = cts_dict.get('coo_exp_par') this_summary.cash_on_hand_end = cts_dict.get('cash_on_hand_end') this_summary.data_source = cts_dict.get('data_source') this_summary.save() except Committee_Time_Summary.DoesNotExist: cts = Committee_Time_Summary(**cts_dict) print "creating committee summary %s" % cts_dict cts.save()
def summarize_noncommittee_periodic_electronic(committee_id, cycle, force_update=True): committee_name = "" try: this_committee = Committee.objects.get(cmte_id=committee_id, cycle=cycle) committee_name = this_committee.cmte_name except Committee.DoesNotExist: print "Missing committee name" pass except Committee.MultipleObjectsReturned: print "multiple committees!! id=%s" % (committee_id) pass this_cycle_calendar = cycle_calendar[int(cycle)] this_cycle_start = this_cycle_calendar['start'] this_cycle_end = this_cycle_calendar['end'] relevant_filings = new_filing.objects.filter( fec_id=committee_id, is_f5_quarterly=True, is_superceded=False, coverage_from_date__gte=this_cycle_start, coverage_to_date__lte=this_cycle_end).order_by('coverage_from_date') #print "processing %s" % committee_id if not relevant_filings: #print "No filings found for %s" % (committee_id) return None # check gaps last_end_date = None for i, this_filing in enumerate(relevant_filings): #print i, this_filing.coverage_from_date, this_filing.coverage_through_date if i == 0: if this_filing.coverage_from_date - this_cycle_start > one_day: pass #print "Missing coverage from start of cycle!!" # set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id) if i > 0: difference = this_filing.coverage_from_date - last_end_date if difference > one_day: pass #print "gap found!" # icpsrset_gap_list(last_end_date,this_filing.coverage_from_date, committee_id) #print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date) last_end_date = this_filing.coverage_to_date form = this_filing.form_type header_data = this_filing.header_data cts_dict = map_summary_form_to_dict(form, header_data) #print "Form = %s cts_dict = %s" % (form, cts_dict) cts_dict['filing_number'] = this_filing.filing_number cts_dict['coverage_through_date'] = this_filing.coverage_to_date cts_dict['coverage_from_date'] = this_filing.coverage_from_date cts_dict['data_source'] = 'electronic' cts_dict['com_id'] = committee_id # only reported receipts *are* the contribs, so... cts_dict['tot_contrib'] = cts_dict['tot_receipts'] cts_dict['com_name'] = committee_name for i in cts_dict: if cts_dict[i] == '': cts_dict[i] = None try: this_summary = Committee_Time_Summary.objects.get( com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date) if force_update: this_summary.filing_number = this_filing.filing_number this_summary.tot_receipts = cts_dict.get('tot_receipts') this_summary.tot_contrib = cts_dict.get('tot_contrib') this_summary.com_name = cts_dict.get('com_name') this_summary.tot_disburse = cts_dict.get('tot_disburse') this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad') this_summary.data_source = cts_dict.get('data_source') this_summary.save() except Committee_Time_Summary.DoesNotExist: cts = Committee_Time_Summary(**cts_dict) cts.save()
def summarize_noncommittee_periodic_electronic(committee_id, force_update=True): committee_name = "" try: this_committee = Committee.objects.get(cmte_id=committee_id, cycle=CYCLE) committee_name = this_committee.cmte_name except Committee.DoesNotExist: print "Missing committee name" pass except Committee.MultipleObjectsReturned: print "multiple committees!! id=%s" % (committee_id) pass relevant_filings = new_filing.objects.filter(fec_id=committee_id, is_f5_quarterly=True, is_superceded=False, coverage_from_date__gte=date(2013,1,1)).order_by('coverage_from_date') #print "processing %s" % committee_id if not relevant_filings: #print "No filings found for %s" % (committee_id) return None # check gaps last_end_date = None for i, this_filing in enumerate(relevant_filings): #print i, this_filing.coverage_from_date, this_filing.coverage_through_date if i==0: if this_filing.coverage_from_date - this_cycle_start > one_day: #print "Missing coverage from start of cycle!!" set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id) if i>0: difference = this_filing.coverage_from_date - last_end_date if difference > one_day: #print "gap found!" set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id) #print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date) last_end_date = this_filing.coverage_to_date form = this_filing.form_type header_data = this_filing.header_data cts_dict = map_summary_form_to_dict(form, header_data) #print "Form = %s cts_dict = %s" % (form, cts_dict) cts_dict['filing_number'] = this_filing.filing_number cts_dict['coverage_through_date'] = this_filing.coverage_to_date cts_dict['coverage_from_date'] = this_filing.coverage_from_date cts_dict['data_source'] = 'electronic' cts_dict['com_id'] = committee_id # only reported receipts *are* the contribs, so... cts_dict['tot_contrib'] = cts_dict['tot_receipts'] cts_dict['com_name'] = committee_name for i in cts_dict: if cts_dict[i] == '': cts_dict[i] = None try: this_summary = Committee_Time_Summary.objects.get(com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date) if force_update: this_summary.filing_number = this_filing.filing_number this_summary.tot_receipts = cts_dict.get('tot_receipts') this_summary.tot_contrib = cts_dict.get('tot_contrib') this_summary.com_name = cts_dict.get('com_name') this_summary.tot_disburse = cts_dict.get('tot_disburse') this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad') this_summary.data_source = cts_dict.get('data_source') this_summary.save() except Committee_Time_Summary.DoesNotExist: cts = Committee_Time_Summary(**cts_dict) cts.save()