def test_sched_a_queue_transactions_failure(self): # Make sure queues are clear before starting self._clear_sched_a_queues() row = self.SchedAFactory(rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime( 2014, 1, 1)) db.session.commit() manage.update_aggregates() manage.refresh_itemized() # Test insert/update failure row.contbr_nm = 'Shelley Adelson' db.session.add(row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 1) db.session.execute('delete from ofec_sched_a_queue_old') db.session.commit() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(old_queue_count, 0) manage.update_aggregates() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == row.sub_id).one() db.session.refresh(search) new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, row.sub_id) self.assertEqual(search.contributor_name, 'Sheldon Adelson')
def test_sched_a_fulltext_trigger(self): # Test create nml_row = self.NmlSchedAFactory(rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime( 2014, 1, 1)) self.FItemReceiptOrExp( sub_id=nml_row.sub_id, rpt_yr=2014, ) db.session.commit() manage.update_aggregates() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id).one() self.assertEqual(search.contributor_name_text, "'adelson':2 'sheldon':1") # Test update nml_row.contbr_nm = 'Shelly Adelson' db.session.add(nml_row) db.session.commit() manage.update_aggregates() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id).one() db.session.refresh(search) self.assertEqual(search.contributor_name_text, "'adelson':2 'shelli':1") # Test delete db.session.delete(nml_row) db.session.commit() manage.update_aggregates() manage.refresh_itemized() self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id).count(), 0, ) # Test sequential writes make_transient(nml_row) db.session.add(nml_row) db.session.commit() db.session.delete(nml_row) db.session.commit() make_transient(nml_row) db.session.add(nml_row) db.session.commit() manage.update_aggregates() manage.refresh_itemized() self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id).count(), 1, )
def test_sched_a_fulltext_trigger(self): # Test create nml_row = self.NmlSchedAFactory( rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime(2014, 1, 1) ) self.FItemReceiptOrExp( sub_id=nml_row.sub_id, rpt_yr=2014, ) db.session.commit() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).one() self.assertEqual(search.contributor_name_text, "'adelson':2 'sheldon':1") # Test update nml_row.contbr_nm = 'Shelly Adelson' db.session.add(nml_row) db.session.commit() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).one() db.session.refresh(search) self.assertEqual(search.contributor_name_text, "'adelson':2 'shelli':1") # Test delete db.session.delete(nml_row) db.session.commit() manage.refresh_itemized() self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).count(), 0, ) # Test sequential writes make_transient(nml_row) db.session.add(nml_row) db.session.commit() db.session.delete(nml_row) db.session.commit() make_transient(nml_row) db.session.add(nml_row) db.session.commit() manage.refresh_itemized() self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).count(), 1, )
def refresh(): """Update incremental aggregates, itemized schedules, materialized views, then slack a notification to the development team. """ manage.logger.info('Starting nightly refresh...') try: manage.refresh_itemized() manage.refresh_materialized() download.clear_bucket() slack_message = '*Success* nightly updates for {0} completed'.format(env.get_credential('NEW_RELIC_APP_NAME')) utils.post_to_slack(slack_message, '#bots') manage.logger.info(slack_message) except Exception as error: manage.logger.exception(error) slack_message = '*ERROR* nightly update failed for {0}. Check logs.'.format(env.get_credential('NEW_RELIC_APP_NAME')) utils.post_to_slack(slack_message, '#bots') manage.logger.exception(error)
def test_update_aggregate_state_existing_null_amount(self): existing = models.ScheduleAByState.query.filter_by( cycle=2016, ).first() total = existing.total count = existing.count self.SchedAFactory( rpt_yr=2015, cmte_id=existing.committee_id, contbr_st=existing.state, contb_receipt_amt=None, contb_receipt_dt=datetime.datetime(2015, 1, 1), receipt_tp='15J', ) db.session.flush() manage.update_aggregates() manage.refresh_itemized() db.session.refresh(existing) self.assertEqual(existing.total, total) self.assertEqual(existing.count, count)
def refresh(): """Update incremental aggregates, itemized schedules, materialized views, then email logs to the development team. """ buffer = io.StringIO() with mail.CaptureLogs(manage.logger, buffer): try: manage.update_functions() manage.update_aggregates() manage.refresh_itemized() manage.update_itemized('e') manage.update_schemas() download.clear_bucket() except Exception as error: manage.logger.exception(error) try: mail.send_mail(buffer) except Exception as error: logger.exception(error)
def test_update_aggregate_purpose_create(self): filing = self.SchedBFactory(rpt_yr=2015, cmte_id='C12345', disb_amt=538, disb_dt=datetime.datetime(2015, 1, 1), disb_desc='CAMPAIGN BUTTONS', form_tp_cd='11') db.session.commit() manage.update_aggregates() manage.refresh_itemized() rows = models.ScheduleBByPurpose.query.filter_by( cycle=2016, committee_id='C12345', purpose='MATERIALS', ).all() self.assertEqual(len(rows), 1) self.assertEqual(rows[0].total, 538) self.assertEqual(rows[0].count, 1) filing.disbursement_description = 'BUMPER STICKERS' db.session.add(filing) db.session.commit() manage.update_aggregates() manage.refresh_itemized() db.session.refresh(rows[0]) self.assertEqual(rows[0].total, 538) self.assertEqual(rows[0].count, 1) filing.disb_desc = 'HANGING OUT' db.session.add(filing) db.session.commit() manage.update_aggregates() manage.refresh_itemized() db.session.refresh(rows[0]) self.assertEqual(rows[0].total, 0) self.assertEqual(rows[0].count, 0)
def test_sched_a_queue_transactions_failure(self): # Make sure queues are clear before starting self._clear_sched_a_queues() nml_row = self.NmlSchedAFactory( rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime(2014, 1, 1) ) self.FItemReceiptOrExp( sub_id=nml_row.sub_id, rpt_yr=2014, ) db.session.commit() manage.refresh_itemized() # Test insert/update failure nml_row.contbr_nm = 'Shelley Adelson' db.session.add(nml_row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 1) db.session.execute('delete from ofec_sched_a_queue_old') db.session.commit() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(old_queue_count, 0) manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).one() db.session.refresh(search) new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, nml_row.sub_id) self.assertEqual(search.contributor_name, 'Sheldon Adelson')
def test_sched_a_queue_transactions_success(self): # Make sure queues are clear before starting self._clear_sched_a_queues() # Test create row = self.SchedAFactory(rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime( 2014, 1, 1)) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 0) manage.update_aggregates() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == row.sub_id).one() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, row.sub_id) # Test update row.contbr_nm = 'Shelly Adelson' db.session.add(row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 1) manage.update_aggregates() manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == row.sub_id).one() db.session.refresh(search) new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, row.sub_id) # Test delete db.session.delete(row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 1) manage.update_aggregates() manage.refresh_itemized() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == row.sub_id).count(), 0, )
def test_sched_a_queue_transactions_success(self): # Make sure queues are clear before starting self._clear_sched_a_queues() # Test create nml_row = self.NmlSchedAFactory( rpt_yr=2014, contbr_nm='Sheldon Adelson', contb_receipt_dt=datetime.datetime(2014, 1, 1) ) self.FItemReceiptOrExp( sub_id=nml_row.sub_id, rpt_yr=2014, ) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 0) manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).one() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, nml_row.sub_id) # Test update nml_row.contbr_nm = 'Shelly Adelson' db.session.add(nml_row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 1) self.assertEqual(old_queue_count, 1) manage.refresh_itemized() search = models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).one() db.session.refresh(search) new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual(search.sub_id, nml_row.sub_id) # Test delete db.session.delete(nml_row) db.session.commit() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 1) manage.refresh_itemized() new_queue_count = self._get_sched_a_queue_new_count() old_queue_count = self._get_sched_a_queue_old_count() self.assertEqual(new_queue_count, 0) self.assertEqual(old_queue_count, 0) self.assertEqual( models.ScheduleA.query.filter( models.ScheduleA.sub_id == nml_row.sub_id ).count(), 0, )