def set_cycle(self, save_now=True):
if self.coverage_from_date:
self.cycle = get_cycle_from_date(self.coverage_from_date)
if save_now:
self.save()
elif self.filed_date:
self.cycle = get_cycle_from_date(self.filed_date)
if save_now:
self.save()
def set_data_from_candidate_id(skedeline, candidate_id):
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
try:
this_candidate = Candidate_Overlay.objects.get(fec_id=candidate_id,
cycle=(THIS_CYCLE))
skedeline.candidate_id_checked = this_candidate.fec_id
skedeline.candidate_checked = this_candidate
skedeline.candidate_district_checked = this_candidate.office_district
skedeline.district_checked = this_candidate.district
skedeline.candidate_office_checked = this_candidate.office
skedeline.candidate_party_checked = this_candidate.party
skedeline.candidate_state_checked = this_candidate.state
skedeline.candidate_name_checked = this_candidate.name
skedeline.support_oppose_checked = skedeline.support_oppose_code
skedeline.save()
return True
except Candidate_Overlay.DoesNotExist:
print "Missing candidate overlay for %s filing %s" % (
candidate_id, skedeline.filing_number)
return False
def fuzzy_match_candidate(skedeline):
state = skedeline.candidate_state
name_to_check = "%s, %s" % (skedeline.candidate_last_name,
skedeline.candidate_first_name)
office = skedeline.candidate_office
state = skedeline.candidate_state
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
result = run_fec_query(name_to_check,
state=state,
office=office,
cycle=THIS_CYCLE,
fuzzy=True)
if result:
if result[0]['match']:
print "Fuzzy matching matched %s, %s, %s to %s with id %s" % (
name_to_check, state, office, result[0]['name'],
result[0]['id'])
return set_data_from_candidate_id(skedeline, result[0]['id'])
print "Fuzzy matching couldn't match %s, %s, %s" % (name_to_check, state,
office)
return False
def get_candidate_url(self):
cycle = get_cycle_from_date(self.effective_date)
if self.candidate_id_checked:
return "/candidate/%s/%s/%s/" % (cycle, slugify(unicode(self.candidate_name_raw())), self.candidate_id_checked)
elif self.candidate_id_number:
return "/candidate/%s/%s/%s/" % (cycle, slugify(unicode(self.candidate_name_raw())), self.candidate_id_number)
else:
return None
def get_candidate_url(self):
cycle = get_cycle_from_date(self.effective_date)
if self.candidate_id_checked:
return "/candidate/%s/%s/%s/" % (
cycle, slugify(unicode(
self.candidate_name_raw())), self.candidate_id_checked)
elif self.candidate_id_number:
return "/candidate/%s/%s/%s/" % (
cycle, slugify(unicode(
self.candidate_name_raw())), self.candidate_id_number)
else:
return None
def attach_committee_to_skedeline(skedeline):
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
try:
co = Committee_Overlay.objects.get(fec_id=skedeline.filer_committee_id_number, cycle=THIS_CYCLE)
skedeline.committee_name = co.name
skedeline.committee_slug = co.slug
skedeline.save()
except Committee_Overlay.DoesNotExist:
print "Missing committee overlay for %s and cycle %s" % (skedeline.filer_committee_id_number, THIS_CYCLE)
def attach_committee_to_skedeline(skedeline):
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
try:
co = Committee_Overlay.objects.get(
fec_id=skedeline.filer_committee_id_number, cycle=THIS_CYCLE)
skedeline.committee_name = co.name
skedeline.committee_slug = co.slug
skedeline.save()
except Committee_Overlay.DoesNotExist:
print "Missing committee overlay for %s and cycle %s" % (
skedeline.filer_committee_id_number, THIS_CYCLE)
def fuzzy_match_candidate(skedeline):
state = skedeline.candidate_state
name_to_check = "%s, %s" % (skedeline.candidate_last_name, skedeline.candidate_first_name)
office = skedeline.candidate_office
state = skedeline.candidate_state
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
result = run_fec_query(name_to_check, state=state, office=office, cycle=THIS_CYCLE, fuzzy=True)
if result:
if result[0]['match']:
print "Fuzzy matching matched %s, %s, %s to %s with id %s" % (name_to_check, state, office, result[0]['name'], result[0]['id'])
return set_data_from_candidate_id(skedeline, result[0]['id'])
print "Fuzzy matching couldn't match %s, %s, %s" % (name_to_check, state, office)
return False
def set_data_from_candidate_id(skedeline, candidate_id):
cycle_date = skedeline.effective_date
THIS_CYCLE = None
if cycle_date:
THIS_CYCLE = get_cycle_from_date(cycle_date)
try:
this_candidate = Candidate_Overlay.objects.get(fec_id=candidate_id, cycle=(THIS_CYCLE))
skedeline.candidate_id_checked = this_candidate.fec_id
skedeline.candidate_checked = this_candidate
skedeline.candidate_district_checked = this_candidate.office_district
skedeline.district_checked = this_candidate.district
skedeline.candidate_office_checked = this_candidate.office
skedeline.candidate_party_checked = this_candidate.party
skedeline.candidate_state_checked = this_candidate.state
skedeline.candidate_name_checked = this_candidate.name
skedeline.support_oppose_checked = skedeline.support_oppose_code
skedeline.save()
return True
except Candidate_Overlay.DoesNotExist:
print "Missing candidate overlay for %s filing %s" % (candidate_id, skedeline.filing_number)
return False
def process_new_filing(thisnewfiling, fp=None, filing_time=None, filing_time_is_exact=False):
""" Enter the file header if needed. """
if not fp:
fp = form_parser()
#print "Processing filing %s" % (filingnum)
f1 = filing(thisnewfiling.filing_number)
if f1.get_error():
return False
form = f1.get_form_type()
version = f1.get_version()
## leave the form if it's already been entered-- that's where it says if it is terminated.
if not thisnewfiling.form_type:
thisnewfiling.form_type = form
# check if it's an amendment based on form types -- if so, mark it. Otherwise the F1's will look like they haven't been amended.
try:
if thisnewfiling.form_type[-1].upper() == 'A':
thisnewfiling.is_amendment = True
except IndexError:
pass
# only parse forms that we're set up to read
if not fp.is_allowed_form(form):
if verbose:
print "Not a parseable form: %s - %s" % (form, thisnewfiling.filing_number)
if thisnewfiling.is_amendment:
thisnewfiling.save()
return True
header = f1.get_first_row()
header_line = fp.parse_form_line(header, version)
amended_filing=None
if f1.is_amendment:
amended_filing = f1.headers['filing_amended']
from_date = None
through_date = None
#print "header line is: %s " % header_line
try:
# dateparse('') will give today, oddly
if header_line['coverage_from_date']:
from_date = dateparse(header_line['coverage_from_date'])
if from_date:
thisnewfiling.cycle = get_cycle_from_date(from_date)
except KeyError:
print "problem with coverage_from_date"
pass
try:
if header_line['coverage_through_date']:
through_date = dateparse(header_line['coverage_through_date'])
if through_date:
thisnewfiling.cycle = get_cycle_from_date(through_date)
except KeyError:
print "coverage_through_date"
pass
# Create the filing -- but don't mark it as being complete.
thisnewfiling.fec_id = f1.headers['fec_id']
thisnewfiling.coverage_from_date = from_date
thisnewfiling.coverage_to_date = through_date
thisnewfiling.is_amendment = f1.is_amendment
thisnewfiling.amends_filing = amended_filing
thisnewfiling.amendment_number = f1.headers['report_number'] or None
thisnewfiling.header_data = header_line
print thisnewfiling.__dict__
thisnewfiling.save()
return True
def enter_filing(data_hash):
filing_created=False
related_committee = None
try:
thisobj = new_filing.objects.get(filing_number=data_hash['filing_number'])
try:
thisobj.filed_date
except AttributeError:
try:
thisobj.filed_date = get_local_time(data_hash['filed_date'])
thisobj.process_time = get_local_time(data_hash['filed_date'])
thisobj.save()
except pytz.exceptions.AmbiguousTimeError:
thisobj.filed_date = data_hash['filed_date']
thisobj.process_time = data_hash['filed_date']
thisobj.save()
except new_filing.DoesNotExist:
print "entering %s %s" % (data_hash['filing_number'], data_hash['committee_id'])
is_superpac=False
try:
thisobj = new_filing.objects.create(
is_superpac = is_superpac,
#related_committee = related_committee,
fec_id = data_hash['committee_id'],
committee_name = data_hash['committee_name'],
filing_number = data_hash['filing_number'],
form_type = data_hash['form_type'],
filed_date = get_local_time(data_hash['filed_date']),
process_time = get_local_time(data_hash['filed_date']),
)
except pytz.exceptions.AmbiguousTimeError:
thisobj = new_filing.objects.create(
is_superpac = is_superpac,
#related_committee = related_committee,
fec_id = data_hash['committee_id'],
committee_name = data_hash['committee_name'],
filing_number = data_hash['filing_number'],
form_type = data_hash['form_type'],
filed_date = data_hash['filed_date'],
process_time = data_hash['filed_date'],
)
filing_created=True
needs_saving=False
try:
thisobj.coverage_from_date = data_hash['coverage_from_date']
thisobj.cycle = get_cycle_from_date(data_hash['coverage_from_date'])
needs_saving=True
except KeyError:
pass
try:
thisobj.coverage_to_date = data_hash['coverage_to_date']
needs_saving=True
except KeyError:
pass
if needs_saving:
thisobj.save()
# return true if a new filing was created
return filing_created
def get_committee_url(self):
cycle = get_cycle_from_date(self.effective_date)
return "/committee/%s/%s/%s/" % (cycle, self.committee_slug, self.filer_committee_id_number)
def enter_filing(data_hash):
filing_created = False
related_committee = None
try:
thisobj = new_filing.objects.get(
filing_number=data_hash['filing_number'])
try:
thisobj.filed_date
except AttributeError:
try:
thisobj.filed_date = get_local_time(data_hash['filed_date'])
thisobj.process_time = get_local_time(data_hash['filed_date'])
thisobj.save()
except pytz.exceptions.AmbiguousTimeError:
thisobj.filed_date = data_hash['filed_date']
thisobj.process_time = data_hash['filed_date']
thisobj.save()
except new_filing.DoesNotExist:
print "entering %s %s" % (data_hash['filing_number'],
data_hash['committee_id'])
is_superpac = False
try:
thisobj = new_filing.objects.create(
is_superpac=is_superpac,
#related_committee = related_committee,
fec_id=data_hash['committee_id'],
committee_name=data_hash['committee_name'],
filing_number=data_hash['filing_number'],
form_type=data_hash['form_type'],
filed_date=get_local_time(data_hash['filed_date']),
process_time=get_local_time(data_hash['filed_date']),
)
except pytz.exceptions.AmbiguousTimeError:
thisobj = new_filing.objects.create(
is_superpac=is_superpac,
#related_committee = related_committee,
fec_id=data_hash['committee_id'],
committee_name=data_hash['committee_name'],
filing_number=data_hash['filing_number'],
form_type=data_hash['form_type'],
filed_date=data_hash['filed_date'],
process_time=data_hash['filed_date'],
)
filing_created = True
needs_saving = False
try:
thisobj.coverage_from_date = data_hash['coverage_from_date']
thisobj.cycle = get_cycle_from_date(
data_hash['coverage_from_date'])
needs_saving = True
except KeyError:
pass
try:
thisobj.coverage_to_date = data_hash['coverage_to_date']
needs_saving = True
except KeyError:
pass
if needs_saving:
thisobj.save()
# return true if a new filing was created
return filing_created
def get_committee_url(self):
cycle = get_cycle_from_date(self.effective_date)
return "/committee/%s/%s/%s/" % (cycle, self.committee_slug,
self.filer_committee_id_number)