def summarize_committee_periodic_electronic(committee_id,
cycle,
force_update=True):
# it's a pain, but we need the committee name in this model.
committee_name = ""
try:
this_committee = Committee.objects.get(cmte_id=committee_id,
cycle=cycle)
committee_name = this_committee.cmte_name
except Committee.DoesNotExist:
print "Missing committee name"
pass
except Committee.MultipleObjectsReturned:
print "multiple committees!! id=%s" % (committee_id)
pass
this_cycle_calendar = cycle_calendar[int(cycle)]
this_cycle_start = this_cycle_calendar['start']
this_cycle_end = this_cycle_calendar['end']
relevant_filings = new_filing.objects.filter(
fec_id=committee_id,
is_superceded=False,
coverage_from_date__gte=this_cycle_start,
coverage_to_date__lte=this_cycle_end,
form_type__in=[
'F3P', 'F3PN', 'F3PA', 'F3PT', 'F3', 'F3A', 'F3N', 'F3T', 'F3X',
'F3XA', 'F3XN', 'F3XT'
]).order_by('coverage_from_date')
#print "processing %s" % committee_id
if not relevant_filings:
#print "No filings found for %s" % (committee_id)
return None
# check gaps
last_end_date = None
for i, this_filing in enumerate(relevant_filings):
#print i, this_filing.coverage_from_date, this_filing.coverage_through_date
if i == 0:
if this_filing.coverage_from_date - this_cycle_start > one_day:
pass
#print "Missing coverage from start of cycle!!"
# set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id)
if i > 0:
difference = this_filing.coverage_from_date - last_end_date
if difference > one_day:
pass
#print "gap found!"
# set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id)
#print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date)
last_end_date = this_filing.coverage_to_date
form = this_filing.form_type
header_data = this_filing.header_data
cts_dict = map_summary_form_to_dict(form, header_data)
#print "Form = %s cts_dict = %s" % (form, cts_dict)
tot_contribs = string_to_float(
cts_dict['tot_ite_contrib']) + string_to_float(
cts_dict['tot_non_ite_contrib'])
cts_dict['filing_number'] = this_filing.filing_number
cts_dict['coverage_through_date'] = this_filing.coverage_to_date
cts_dict['coverage_from_date'] = this_filing.coverage_from_date
cts_dict['data_source'] = 'electronic'
cts_dict['com_id'] = committee_id
cts_dict['tot_contrib'] = tot_contribs
cts_dict['com_name'] = committee_name
for i in cts_dict:
if cts_dict[i] == '':
cts_dict[i] = None
try:
this_summary = Committee_Time_Summary.objects.get(
com_id=committee_id,
coverage_from_date=this_filing.coverage_from_date,
coverage_through_date=this_filing.coverage_to_date)
if force_update:
this_summary.filing_number = this_filing.filing_number
this_summary.tot_receipts = cts_dict.get('tot_receipts')
this_summary.tot_ite_contrib = cts_dict.get('tot_ite_contrib')
this_summary.tot_non_ite_contrib = cts_dict.get(
'tot_non_ite_contrib')
this_summary.tot_contrib = cts_dict.get('tot_contrib')
this_summary.com_name = cts_dict.get('com_name')
this_summary.tot_disburse = cts_dict.get('tot_disburse')
this_summary.new_loans = cts_dict.get('new_loans')
this_summary.outstanding_loans = cts_dict.get(
'outstanding_loans')
this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad')
this_summary.coo_exp_par = cts_dict.get('coo_exp_par')
this_summary.cash_on_hand_end = cts_dict.get(
'cash_on_hand_end')
this_summary.data_source = cts_dict.get('data_source')
this_summary.save()
except Committee_Time_Summary.DoesNotExist:
cts = Committee_Time_Summary(**cts_dict)
print "creating committee summary %s" % cts_dict
cts.save()
def summarize_committee_periodic_electronic(committee_id, force_update=True):
# it's a pain, but we need the committee name in this model.
committee_name = ""
try:
this_committee = Committee.objects.get(cmte_id=committee_id, cycle=CYCLE)
committee_name = this_committee.cmte_name
except Committee.DoesNotExist:
print "Missing committee name"
pass
except Committee.MultipleObjectsReturned:
print "multiple committees!! id=%s" % (committee_id)
pass
relevant_filings = new_filing.objects.filter(fec_id=committee_id, is_superceded=False, coverage_from_date__gte=date(2013,1,1), form_type__in=['F3P', 'F3PN', 'F3PA', 'F3PT', 'F3', 'F3A', 'F3N', 'F3T', 'F3X', 'F3XA', 'F3XN', 'F3XT']).order_by('coverage_from_date')
#print "processing %s" % committee_id
if not relevant_filings:
#print "No filings found for %s" % (committee_id)
return None
# check gaps
last_end_date = None
for i, this_filing in enumerate(relevant_filings):
#print i, this_filing.coverage_from_date, this_filing.coverage_through_date
if i==0:
if this_filing.coverage_from_date - this_cycle_start > one_day:
#print "Missing coverage from start of cycle!!"
set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id)
if i>0:
difference = this_filing.coverage_from_date - last_end_date
if difference > one_day:
#print "gap found!"
set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id)
#print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date)
last_end_date = this_filing.coverage_to_date
form = this_filing.form_type
header_data = this_filing.header_data
cts_dict = map_summary_form_to_dict(form, header_data)
#print "Form = %s cts_dict = %s" % (form, cts_dict)
tot_contribs = string_to_float(cts_dict['tot_ite_contrib']) + string_to_float(cts_dict['tot_non_ite_contrib'])
cts_dict['filing_number'] = this_filing.filing_number
cts_dict['coverage_through_date'] = this_filing.coverage_to_date
cts_dict['coverage_from_date'] = this_filing.coverage_from_date
cts_dict['data_source'] = 'electronic'
cts_dict['com_id'] = committee_id
cts_dict['tot_contrib'] = tot_contribs
cts_dict['com_name'] = committee_name
for i in cts_dict:
if cts_dict[i] == '':
cts_dict[i] = None
try:
this_summary = Committee_Time_Summary.objects.get(com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date)
if force_update:
this_summary.filing_number = this_filing.filing_number
this_summary.tot_receipts = cts_dict.get('tot_receipts')
this_summary.tot_ite_contrib = cts_dict.get('tot_ite_contrib')
this_summary.tot_non_ite_contrib = cts_dict.get('tot_non_ite_contrib')
this_summary.tot_contrib = cts_dict.get('tot_contrib')
this_summary.com_name = cts_dict.get('com_name')
this_summary.tot_disburse = cts_dict.get('tot_disburse')
this_summary.new_loans = cts_dict.get('new_loans')
this_summary.outstanding_loans = cts_dict.get('outstanding_loans')
this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad')
this_summary.coo_exp_par = cts_dict.get('coo_exp_par')
this_summary.cash_on_hand_end = cts_dict.get('cash_on_hand_end')
this_summary.data_source = cts_dict.get('data_source')
this_summary.save()
except Committee_Time_Summary.DoesNotExist:
cts = Committee_Time_Summary(**cts_dict)
print "creating committee summary %s" % cts_dict
cts.save()
def summarize_noncommittee_periodic_electronic(committee_id,
cycle,
force_update=True):
committee_name = ""
try:
this_committee = Committee.objects.get(cmte_id=committee_id,
cycle=cycle)
committee_name = this_committee.cmte_name
except Committee.DoesNotExist:
print "Missing committee name"
pass
except Committee.MultipleObjectsReturned:
print "multiple committees!! id=%s" % (committee_id)
pass
this_cycle_calendar = cycle_calendar[int(cycle)]
this_cycle_start = this_cycle_calendar['start']
this_cycle_end = this_cycle_calendar['end']
relevant_filings = new_filing.objects.filter(
fec_id=committee_id,
is_f5_quarterly=True,
is_superceded=False,
coverage_from_date__gte=this_cycle_start,
coverage_to_date__lte=this_cycle_end).order_by('coverage_from_date')
#print "processing %s" % committee_id
if not relevant_filings:
#print "No filings found for %s" % (committee_id)
return None
# check gaps
last_end_date = None
for i, this_filing in enumerate(relevant_filings):
#print i, this_filing.coverage_from_date, this_filing.coverage_through_date
if i == 0:
if this_filing.coverage_from_date - this_cycle_start > one_day:
pass
#print "Missing coverage from start of cycle!!"
# set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id)
if i > 0:
difference = this_filing.coverage_from_date - last_end_date
if difference > one_day:
pass
#print "gap found!"
# icpsrset_gap_list(last_end_date,this_filing.coverage_from_date, committee_id)
#print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date)
last_end_date = this_filing.coverage_to_date
form = this_filing.form_type
header_data = this_filing.header_data
cts_dict = map_summary_form_to_dict(form, header_data)
#print "Form = %s cts_dict = %s" % (form, cts_dict)
cts_dict['filing_number'] = this_filing.filing_number
cts_dict['coverage_through_date'] = this_filing.coverage_to_date
cts_dict['coverage_from_date'] = this_filing.coverage_from_date
cts_dict['data_source'] = 'electronic'
cts_dict['com_id'] = committee_id
# only reported receipts *are* the contribs, so...
cts_dict['tot_contrib'] = cts_dict['tot_receipts']
cts_dict['com_name'] = committee_name
for i in cts_dict:
if cts_dict[i] == '':
cts_dict[i] = None
try:
this_summary = Committee_Time_Summary.objects.get(
com_id=committee_id,
coverage_from_date=this_filing.coverage_from_date,
coverage_through_date=this_filing.coverage_to_date)
if force_update:
this_summary.filing_number = this_filing.filing_number
this_summary.tot_receipts = cts_dict.get('tot_receipts')
this_summary.tot_contrib = cts_dict.get('tot_contrib')
this_summary.com_name = cts_dict.get('com_name')
this_summary.tot_disburse = cts_dict.get('tot_disburse')
this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad')
this_summary.data_source = cts_dict.get('data_source')
this_summary.save()
except Committee_Time_Summary.DoesNotExist:
cts = Committee_Time_Summary(**cts_dict)
cts.save()
def summarize_noncommittee_periodic_electronic(committee_id, force_update=True):
committee_name = ""
try:
this_committee = Committee.objects.get(cmte_id=committee_id, cycle=CYCLE)
committee_name = this_committee.cmte_name
except Committee.DoesNotExist:
print "Missing committee name"
pass
except Committee.MultipleObjectsReturned:
print "multiple committees!! id=%s" % (committee_id)
pass
relevant_filings = new_filing.objects.filter(fec_id=committee_id, is_f5_quarterly=True, is_superceded=False, coverage_from_date__gte=date(2013,1,1)).order_by('coverage_from_date')
#print "processing %s" % committee_id
if not relevant_filings:
#print "No filings found for %s" % (committee_id)
return None
# check gaps
last_end_date = None
for i, this_filing in enumerate(relevant_filings):
#print i, this_filing.coverage_from_date, this_filing.coverage_through_date
if i==0:
if this_filing.coverage_from_date - this_cycle_start > one_day:
#print "Missing coverage from start of cycle!!"
set_gap_list(this_cycle_start,this_filing.coverage_from_date, committee_id)
if i>0:
difference = this_filing.coverage_from_date - last_end_date
if difference > one_day:
#print "gap found!"
set_gap_list(last_end_date,this_filing.coverage_from_date, committee_id)
#print "Got filing %s - %s" % (this_filing.coverage_from_date, this_filing.coverage_through_date)
last_end_date = this_filing.coverage_to_date
form = this_filing.form_type
header_data = this_filing.header_data
cts_dict = map_summary_form_to_dict(form, header_data)
#print "Form = %s cts_dict = %s" % (form, cts_dict)
cts_dict['filing_number'] = this_filing.filing_number
cts_dict['coverage_through_date'] = this_filing.coverage_to_date
cts_dict['coverage_from_date'] = this_filing.coverage_from_date
cts_dict['data_source'] = 'electronic'
cts_dict['com_id'] = committee_id
# only reported receipts *are* the contribs, so...
cts_dict['tot_contrib'] = cts_dict['tot_receipts']
cts_dict['com_name'] = committee_name
for i in cts_dict:
if cts_dict[i] == '':
cts_dict[i] = None
try:
this_summary = Committee_Time_Summary.objects.get(com_id=committee_id, coverage_from_date=this_filing.coverage_from_date, coverage_through_date=this_filing.coverage_to_date)
if force_update:
this_summary.filing_number = this_filing.filing_number
this_summary.tot_receipts = cts_dict.get('tot_receipts')
this_summary.tot_contrib = cts_dict.get('tot_contrib')
this_summary.com_name = cts_dict.get('com_name')
this_summary.tot_disburse = cts_dict.get('tot_disburse')
this_summary.ind_exp_mad = cts_dict.get('ind_exp_mad')
this_summary.data_source = cts_dict.get('data_source')
this_summary.save()
except Committee_Time_Summary.DoesNotExist:
cts = Committee_Time_Summary(**cts_dict)
cts.save()