def ingest(filepath):
'''Ingest file into database'''
print "Ingesting %s" % filepath
rows = parse_fec_file(filepath)
# check history table to see if this file is done
with db.transaction():
for idx, row in enumerate(rows):
print "Checking row %d of %d from %s" % (idx, len(rows), filepath)
try:
contribution_in_db = Contribution.get(cycle=row['cycle'], sub_id=row['sub_id'])
except Contribution.DoesNotExist:
contribution_in_db = None
# If the row isn't already there, insert it
if not contribution_in_db:
print t.cyan("\tInserting new row %d of %s" % (idx, filepath))
new_contribution = Contribution.create(**row)
ContributionHistory.create(contribution=new_contribution.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
# If the row is there, check for modifications
else:
# If it has not been modified, simply add a ContributionHistory object
contribution_in_db_dict = get_dictionary_from_model(contribution_in_db)
# x = {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]}
# y = {k:v for k,v in row.iteritems() if k != "date"}
if {k:v for k,v in contribution_in_db_dict.iteritems() if k not in ["date", "id"]} == {k:v for k,v in row.iteritems() if k != "date"}:
print t.white("\tNo changes found in row %d of %s" % (idx, filepath))
ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
# If it has been modified, create a new object and give the new object a contribution history
else:
print t.magenta("\tDetected change in row %d of %s" % (idx, filepath))
# print diff(x,y)
# import pdb; pdb.set_trace()
ContributionChanges.create(contribution=contribution_in_db.id, **{k:v for k,v in contribution_in_db_dict.iteritems() if k != "id"})
for k,v in row.iteritems():
if v != getattr(contribution_in_db, k):
setattr(contribution_in_db, k, v)
contribution_in_db.save()
ContributionHistory.create(contribution=contribution_in_db.id, date=row['date'], cycle=row['cycle'], sub_id=row['sub_id'])
myfile, _ = File.get_or_create(
name = os.path.basename(filepath),
years=next(re.finditer(r'\d{4}_\d{4}', os.path.basename(filepath))),
sha1 = sha1OfFile(filepath),
updated = dateparse(os.path.dirname(filepath).split("/")[-1].replace("downloaded_", "").replace("_", "-")).date(),
ingested = True
)
def hello():
before = dateparse(request.args.get('before')).date()
after = dateparse(request.args.get('after')).date()
ret = []
for contrib in ContributionChanges.select().where(ContributionChanges.date == before):
before_contrib_dict = get_dictionary_from_model(contrib)
after_contrib_dict = get_dictionary_from_model(Contribution.get(sub_id=contrib.sub_id).get_on_date(after))
before_contrib_dict.pop('id')
before_contrib_dict.pop('contribution')
after_contrib_dict.pop('id')
after_contrib_dict['transaction_pgi'] = transaction_pgi_dict[after_contrib_dict['transaction_pgi']] if after_contrib_dict['transaction_pgi'] else None
before_contrib_dict['transaction_pgi'] = transaction_pgi_dict[before_contrib_dict['transaction_pgi']] if before_contrib_dict['transaction_pgi'] else None
after_contrib_dict['ammendment_id'] = ammendment_id_dict[after_contrib_dict['ammendment_id']] if after_contrib_dict['ammendment_id'] else None
before_contrib_dict['ammendment_id'] = ammendment_id_dict[before_contrib_dict['ammendment_id']] if before_contrib_dict['ammendment_id'] else None
after_contrib_dict['transaction_tp'] = transaction_tp_dict[after_contrib_dict['transaction_tp']] if after_contrib_dict['transaction_tp'] else None
before_contrib_dict['transaction_tp'] = transaction_tp_dict[before_contrib_dict['transaction_tp']] if before_contrib_dict['transaction_tp'] else None
after_contrib_dict['report_type'] = report_type_dict[after_contrib_dict['report_type']] if after_contrib_dict['report_type'] else None
before_contrib_dict['report_type'] = report_type_dict[before_contrib_dict['report_type']] if before_contrib_dict['report_type'] else None
# report_type_dict
ret.append({
"before": before_contrib_dict,
"after": after_contrib_dict,
"changes": list(set([x[1][0][0] for x in diff(before_contrib_dict, after_contrib_dict).diffs if x[0] not in ["equal", "context_end_container"] and x[1][0][0] not in ['contribution', 'date', 'id']]))
})
before_sub_ids = set([x.sub_id for x in ContributionHistory.select().where(ContributionHistory.date == before)])
after_sub_ids = set([x.sub_id for x in ContributionHistory.select().where(ContributionHistory.date == after)])
for sub_id in (before_sub_ids - after_sub_ids):
ret.append({
"before": get_dictionary_from_model(Contribution.get(sub_id=sub_id)),
"after": None,
"changes": None
})
return render_template('diff.html', ret=ret, before=before, after=after)