def test_sched_a_queue_transactions_failure(self):
# Make sure queues are clear before starting
self._clear_sched_a_queues()
row = self.SchedAFactory(rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(
2014, 1, 1))
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
# Test insert/update failure
row.contbr_nm = 'Shelley Adelson'
db.session.add(row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 1)
db.session.execute('delete from ofec_sched_a_queue_old')
db.session.commit()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(old_queue_count, 0)
manage.update_aggregates()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == row.sub_id).one()
db.session.refresh(search)
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, row.sub_id)
self.assertEqual(search.contributor_name, 'Sheldon Adelson')
def test_sched_a_fulltext_trigger(self):
# Test create
nml_row = self.NmlSchedAFactory(rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(
2014, 1, 1))
self.FItemReceiptOrExp(
sub_id=nml_row.sub_id,
rpt_yr=2014,
)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id).one()
self.assertEqual(search.contributor_name_text,
"'adelson':2 'sheldon':1")
# Test update
nml_row.contbr_nm = 'Shelly Adelson'
db.session.add(nml_row)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id).one()
db.session.refresh(search)
self.assertEqual(search.contributor_name_text,
"'adelson':2 'shelli':1")
# Test delete
db.session.delete(nml_row)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id).count(),
0,
)
# Test sequential writes
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
db.session.delete(nml_row)
db.session.commit()
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id).count(),
1,
)
def test_sched_a_fulltext_trigger(self):
# Test create
nml_row = self.NmlSchedAFactory(
rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(2014, 1, 1)
)
self.FItemReceiptOrExp(
sub_id=nml_row.sub_id,
rpt_yr=2014,
)
db.session.commit()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
self.assertEqual(search.contributor_name_text, "'adelson':2 'sheldon':1")
# Test update
nml_row.contbr_nm = 'Shelly Adelson'
db.session.add(nml_row)
db.session.commit()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
db.session.refresh(search)
self.assertEqual(search.contributor_name_text, "'adelson':2 'shelli':1")
# Test delete
db.session.delete(nml_row)
db.session.commit()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).count(),
0,
)
# Test sequential writes
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
db.session.delete(nml_row)
db.session.commit()
make_transient(nml_row)
db.session.add(nml_row)
db.session.commit()
manage.refresh_itemized()
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).count(),
1,
)
def refresh():
"""Update incremental aggregates, itemized schedules, materialized views,
then slack a notification to the development team.
"""
manage.logger.info('Starting nightly refresh...')
try:
manage.refresh_itemized()
manage.refresh_materialized()
download.clear_bucket()
slack_message = '*Success* nightly updates for {0} completed'.format(env.get_credential('NEW_RELIC_APP_NAME'))
utils.post_to_slack(slack_message, '#bots')
manage.logger.info(slack_message)
except Exception as error:
manage.logger.exception(error)
slack_message = '*ERROR* nightly update failed for {0}. Check logs.'.format(env.get_credential('NEW_RELIC_APP_NAME'))
utils.post_to_slack(slack_message, '#bots')
manage.logger.exception(error)
def test_update_aggregate_state_existing_null_amount(self):
existing = models.ScheduleAByState.query.filter_by(
cycle=2016, ).first()
total = existing.total
count = existing.count
self.SchedAFactory(
rpt_yr=2015,
cmte_id=existing.committee_id,
contbr_st=existing.state,
contb_receipt_amt=None,
contb_receipt_dt=datetime.datetime(2015, 1, 1),
receipt_tp='15J',
)
db.session.flush()
manage.update_aggregates()
manage.refresh_itemized()
db.session.refresh(existing)
self.assertEqual(existing.total, total)
self.assertEqual(existing.count, count)
def refresh():
"""Update incremental aggregates, itemized schedules, materialized views,
then email logs to the development team.
"""
buffer = io.StringIO()
with mail.CaptureLogs(manage.logger, buffer):
try:
manage.update_functions()
manage.update_aggregates()
manage.refresh_itemized()
manage.update_itemized('e')
manage.update_schemas()
download.clear_bucket()
except Exception as error:
manage.logger.exception(error)
try:
mail.send_mail(buffer)
except Exception as error:
logger.exception(error)
def test_update_aggregate_purpose_create(self):
filing = self.SchedBFactory(rpt_yr=2015,
cmte_id='C12345',
disb_amt=538,
disb_dt=datetime.datetime(2015, 1, 1),
disb_desc='CAMPAIGN BUTTONS',
form_tp_cd='11')
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
rows = models.ScheduleBByPurpose.query.filter_by(
cycle=2016,
committee_id='C12345',
purpose='MATERIALS',
).all()
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0].total, 538)
self.assertEqual(rows[0].count, 1)
filing.disbursement_description = 'BUMPER STICKERS'
db.session.add(filing)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
db.session.refresh(rows[0])
self.assertEqual(rows[0].total, 538)
self.assertEqual(rows[0].count, 1)
filing.disb_desc = 'HANGING OUT'
db.session.add(filing)
db.session.commit()
manage.update_aggregates()
manage.refresh_itemized()
db.session.refresh(rows[0])
self.assertEqual(rows[0].total, 0)
self.assertEqual(rows[0].count, 0)
def test_sched_a_queue_transactions_failure(self):
# Make sure queues are clear before starting
self._clear_sched_a_queues()
nml_row = self.NmlSchedAFactory(
rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(2014, 1, 1)
)
self.FItemReceiptOrExp(
sub_id=nml_row.sub_id,
rpt_yr=2014,
)
db.session.commit()
manage.refresh_itemized()
# Test insert/update failure
nml_row.contbr_nm = 'Shelley Adelson'
db.session.add(nml_row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 1)
db.session.execute('delete from ofec_sched_a_queue_old')
db.session.commit()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(old_queue_count, 0)
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
db.session.refresh(search)
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, nml_row.sub_id)
self.assertEqual(search.contributor_name, 'Sheldon Adelson')
def test_sched_a_queue_transactions_success(self):
# Make sure queues are clear before starting
self._clear_sched_a_queues()
# Test create
row = self.SchedAFactory(rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(
2014, 1, 1))
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 0)
manage.update_aggregates()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == row.sub_id).one()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, row.sub_id)
# Test update
row.contbr_nm = 'Shelly Adelson'
db.session.add(row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 1)
manage.update_aggregates()
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == row.sub_id).one()
db.session.refresh(search)
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, row.sub_id)
# Test delete
db.session.delete(row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 1)
manage.update_aggregates()
manage.refresh_itemized()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == row.sub_id).count(),
0,
)
def test_sched_a_queue_transactions_success(self):
# Make sure queues are clear before starting
self._clear_sched_a_queues()
# Test create
nml_row = self.NmlSchedAFactory(
rpt_yr=2014,
contbr_nm='Sheldon Adelson',
contb_receipt_dt=datetime.datetime(2014, 1, 1)
)
self.FItemReceiptOrExp(
sub_id=nml_row.sub_id,
rpt_yr=2014,
)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 0)
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, nml_row.sub_id)
# Test update
nml_row.contbr_nm = 'Shelly Adelson'
db.session.add(nml_row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 1)
self.assertEqual(old_queue_count, 1)
manage.refresh_itemized()
search = models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).one()
db.session.refresh(search)
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(search.sub_id, nml_row.sub_id)
# Test delete
db.session.delete(nml_row)
db.session.commit()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 1)
manage.refresh_itemized()
new_queue_count = self._get_sched_a_queue_new_count()
old_queue_count = self._get_sched_a_queue_old_count()
self.assertEqual(new_queue_count, 0)
self.assertEqual(old_queue_count, 0)
self.assertEqual(
models.ScheduleA.query.filter(
models.ScheduleA.sub_id == nml_row.sub_id
).count(),
0,
)
