Esempio n. 1
0
    def test_update_user(self):
        config_manager = self._setup_simple_config()
        with config_manager.context() as config:
            job = automatic_emails.AutomaticEmailsCronApp(config, '')
            now = utc_now().isoformat()

            es = SuperS().es(
                urls=config.elasticsearch.elasticsearch_urls,
                timeout=config.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(
                config.elasticsearch.elasticsearch_emails_index)
            search = search.doctypes('emails')

            connection = es.get_es()

            job.update_user('*****@*****.**', now, connection)
            connection.refresh()

            s = search.filter(_id='*****@*****.**')
            res = list(s.values_list('last_sending'))

            eq_(len(res), 1)
            eq_(res[0][0], now)

            # Test with a non-existing user
            job.update_user('*****@*****.**', now, connection)
            connection.refresh()

            s = search.filter(_id='*****@*****.**')
            res = list(s.values_list('last_sending'))

            eq_(len(res), 1)
            eq_(res[0][0], now)
Esempio n. 2
0
    def test_update_user(self):
        config_manager = self._setup_simple_config()
        with config_manager.context() as config:
            job = automatic_emails.AutomaticEmailsCronApp(config, '')
            now = utc_now().isoformat()

            es = SuperS().es(
                urls=config.elasticsearch.elasticsearch_urls,
                timeout=config.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(
                config.elasticsearch.elasticsearch_emails_index
            )
            search = search.doctypes('emails')

            connection = es.get_es()

            job.update_user('*****@*****.**', now, connection)
            connection.refresh()

            s = search.filter(_id='*****@*****.**')
            res = list(s.values_list('last_sending'))

            self.assertEqual(len(res), 1)
            self.assertEqual(res[0][0], now)

            # Test with a non-existing user
            job.update_user('*****@*****.**', now, connection)
            connection.refresh()

            s = search.filter(_id='*****@*****.**')
            res = list(s.values_list('last_sending'))

            self.assertEqual(len(res), 1)
            self.assertEqual(res[0][0], now)
Esempio n. 3
0
    def test_cron_job(self, exacttarget_mock):
        config_manager = self._setup_config_manager()
        et_mock = exacttarget_mock.return_value

        # Make get_subscriber raise an exception
        list_service = et_mock.list.return_value = mock.Mock()
        list_service.get_subscriber = mock.Mock(
            side_effect=exacttarget.NewsletterException()
        )

        with config_manager.context() as config:
            tab = crontabber.CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert not information['automatic-emails']['last_error']
            assert information['automatic-emails']['last_success']
            self.assertEqual(et_mock.trigger_send.call_count, 4)

            last_email = u'z\[email protected]'

            # Verify the last call to trigger_send
            fields = {
                'EMAIL_ADDRESS_': last_email,
                'EMAIL_FORMAT_': 'H',
                'TOKEN': last_email
            }

            et_mock.trigger_send.assert_called_with('socorro_dev_test', fields)

            # Verify that user's data was updated
            conf = config.crontabber['class-AutomaticEmailsCronApp']
            es = SuperS().es(
                urls=conf.elasticsearch.elasticsearch_urls,
                timeout=conf.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
            search = search.doctypes('emails')
            es.get_es().refresh()

            emails_list = (
                '*****@*****.**',
                '"Quidam" <*****@*****.**>',
                '*****@*****.**'
            )
            search = search.filter(_id__in=emails_list)
            res = search.values_list('last_sending')
            self.assertEqual(len(res), 3)
            now = utc_now()
            for row in res:
                date = string_to_datetime(row[0])
                self.assertEqual(date.year, now.year)
                self.assertEqual(date.month, now.month)
                self.assertEqual(date.day, now.day)
Esempio n. 4
0
    def test_cron_job(self, exacttarget_mock):
        config_manager = self._setup_config_manager()
        et_mock = exacttarget_mock.return_value

        # Make get_subscriber raise an exception
        list_service = et_mock.list.return_value = mock.Mock()
        list_service.get_subscriber = mock.Mock(
            side_effect=exacttarget.NewsletterException()
        )

        with config_manager.context() as config:
            tab = crontabber.CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert not information['automatic-emails']['last_error']
            assert information['automatic-emails']['last_success']
            self.assertEqual(et_mock.trigger_send.call_count, 4)

            last_email = u'z\[email protected]'

            # Verify the last call to trigger_send
            fields = {
                'EMAIL_ADDRESS_': last_email,
                'EMAIL_FORMAT_': 'H',
                'TOKEN': last_email
            }

            et_mock.trigger_send.assert_called_with('socorro_dev_test', fields)

            # Verify that user's data was updated
            conf = config.crontabber['class-AutomaticEmailsCronApp']
            es = SuperS().es(
                urls=conf.elasticsearch.elasticsearch_urls,
                timeout=conf.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
            search = search.doctypes('emails')
            es.get_es().refresh()

            emails_list = (
                '*****@*****.**',
                '"Quidam" <*****@*****.**>',
                '*****@*****.**'
            )
            search = search.filter(_id__in=emails_list)
            res = search.values_list('last_sending')
            self.assertEqual(len(res), 3)
            now = utc_now()
            for row in res:
                date = string_to_datetime(row[0])
                self.assertEqual(date.year, now.year)
                self.assertEqual(date.month, now.month)
                self.assertEqual(date.day, now.day)
    def run(self, run_datetime):
        logger = self.config.logger

        if self.config.test_mode:
            logger.warning('You are running Automatic Emails cron app '
                           'in test mode')

        delay = datetime.timedelta(days=self.config.delay_between_emails)
        params = {
            'start_date': run_datetime - datetime.timedelta(hours=1),
            'end_date': run_datetime,
            'delayed_date': run_datetime - delay,
            'products': tuple(self.config.restrict_products)
        }

        # Find the indexes to use to optimize the elasticsearch query.
        indexes = self.generate_list_of_indexes(
            params['start_date'],
            params['end_date'],
            self.config.elasticsearch.elasticsearch_index
        )

        # Create and configure the search object.
        connection = SuperS().es(
            urls=self.config.elasticsearch.elasticsearch_urls,
            timeout=self.config.elasticsearch.elasticsearch_timeout,
        )
        search = (
            connection.indexes(*indexes).doctypes(
                self.config.elasticsearch.elasticsearch_doctype
            ).order_by('processed_crash.email')
        )

        # Create filters.
        args_and = {
            'processed_crash.date_processed__lt': params['end_date'],
            'processed_crash.date_processed__gt': params['start_date'],
            'processed_crash.product': [x.lower() for x in params['products']],
        }
        args_not = {
            'processed_crash.email__missing': None,
        }

        filters = elasticutils.F(**args_and)
        filters &= ~elasticutils.F(**args_not)

        search = search.filter(filters)
        count = search.count()  # Total number of results.
        search = search[:count]

        # Get the recently sent emails
        emails = self.get_list_of_emails(params, connection)

        validation_rules = TransformRuleSystem()
        validation_rules.load_rules((
            (verify_email, (), {}, sanitize_email, (), {}),
            (verify_email, (), {}, False, (), {}),
            (
                verify_email_last_sending, (), {'emails_list': emails},
                True, (), {}
            ),
        ))

        template_rules = TransformRuleSystem()
        template_rules.load_rules((
            (
                verify_support_classification, ('bitguard',), {},
                set_email_template, ('socorro_bitguard_en',), {}
            ),
            # If no other rule passed, fall back to the default template.
            (
                True, (), {},
                set_email_template, (self.config.email_template,), {}
            ),
        ))

        for hit in search.values_dict(
            'processed_crash.email',
            'processed_crash.classifications.support.classification',
        ):
            res = validation_rules.apply_until_predicate_fails(hit)

            if res is None:  # All predicates succeeded!
                # Now apply all template rules to find which email template
                # to use.
                template_rules.apply_until_action_succeeds(hit)

                if not hit['email_template']:
                    # Bug 965610 - If the email template is empty, do not send
                    # an email. Setting the default email template to '' means
                    # no generic email will be sent anymore.
                    continue

                email = hit['processed_crash.email']
                self.send_email(hit)
                self.update_user(email, run_datetime, connection.get_es())
                emails[email] = run_datetime
                # logger.info('Automatic Email sent to %s', email)

        # Make sure the next run will have updated data, to avoid sending an
        # email several times.
        connection.get_es().refresh()
Esempio n. 6
0
    def run(self, run_datetime):
        logger = self.config.logger

        if self.config.test_mode:
            logger.warning('You are running Automatic Emails cron app '
                           'in test mode')

        delay = datetime.timedelta(days=self.config.delay_between_emails)
        params = {
            'start_date': run_datetime - datetime.timedelta(hours=1),
            'end_date': run_datetime,
            'delayed_date': run_datetime - delay,
            'products': tuple(self.config.restrict_products)
        }

        # Find the indexes to use to optimize the elasticsearch query.
        indexes = self.generate_list_of_indexes(
            params['start_date'],
            params['end_date'],
            self.config.elasticsearch.elasticsearch_index
        )

        # Create and configure the search object.
        connection = SuperS().es(
            urls=self.config.elasticsearch.elasticsearch_urls,
            timeout=self.config.elasticsearch.elasticsearch_timeout,
        )
        search = (connection.indexes(*indexes)
                            .doctypes(
                                self.config.elasticsearch.elasticsearch_doctype
                            )
                            .order_by('processed_crash.email'))

        # Create filters.
        args_and = {
            'processed_crash.date_processed__lt': params['end_date'],
            'processed_crash.date_processed__gt': params['start_date'],
            'processed_crash.product': [x.lower() for x in params['products']],
        }
        args_not = {
            'processed_crash.email__missing': None,
        }

        filters = elasticutils.F(**args_and)
        filters &= ~elasticutils.F(**args_not)

        search = search.filter(filters)
        count = search.count()  # Total number of results.
        search = search[:count]

        # Get the recently sent emails
        emails = self.get_list_of_emails(params, connection)

        validation_rules = TransformRuleSystem()
        validation_rules.load_rules((
            (verify_email, (), {}, sanitize_email, (), {}),
            (verify_email, (), {}, False, (), {}),
            (
                verify_email_last_sending, (), {'emails_list': emails},
                True, (), {}
            ),
        ))

        template_rules = TransformRuleSystem()
        template_rules.load_rules((
            (
                verify_support_classification, ('bitguard',), {},
                set_email_template, ('socorro_bitguard_en',), {}
            ),
            # If no other rule passed, fall back to the default template.
            (
                True, (), {},
                set_email_template, (self.config.email_template,), {}
            ),
        ))

        for hit in search.values_dict(
            'processed_crash.email',
            'processed_crash.classifications.support.classification',
        ):
            res = validation_rules.apply_until_predicate_fails(hit)

            if res is None:  # All predicates succeeded!
                # Now apply all template rules to find which email template
                # to use.
                template_rules.apply_until_action_succeeds(hit)

                email = hit['processed_crash.email']
                self.send_email(hit)
                self.update_user(email, run_datetime, connection.get_es())
                emails[email] = run_datetime
                # logger.info('Automatic Email sent to %s', email)

        # Make sure the next run will have updated data, to avoid sending an
        # email several times.
        connection.get_es().refresh()
Esempio n. 7
0
    def test_email_cannot_be_sent_twice(self, exacttarget_mock):
        config_manager = self._setup_config_manager(
            restrict_products=['NightlyTrain'])
        et_mock = exacttarget_mock.return_value

        # Prepare failures
        _failures = []
        _email_sent = []

        class SomeRandomError(Exception):
            pass

        def trigger_send(template, fields):
            email = fields['EMAIL_ADDRESS_']
            if email == '*****@*****.**' and email not in _failures:
                _failures.append(email)
                raise SomeRandomError('This is an error. ')
            else:
                _email_sent.append(email)

        et_mock.trigger_send = trigger_send

        with config_manager.context() as config:
            tab = CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert information['automatic-emails']['last_error']
            eq_(information['automatic-emails']['last_error']['type'],
                str(SomeRandomError))

            # Verify that user's data was updated, but not all of it
            eq_(_email_sent, ['*****@*****.**', '*****@*****.**'])
            emails_list = ('*****@*****.**', '*****@*****.**', '*****@*****.**',
                           '*****@*****.**', '*****@*****.**')

            conf = config.crontabber['class-AutomaticEmailsCronApp']
            es = SuperS().es(
                urls=conf.elasticsearch.elasticsearch_urls,
                timeout=conf.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(conf.elasticsearch.elasticsearch_emails_index)
            search = search.doctypes('emails')
            es.get_es().refresh()

            search = search.filter(_id__in=emails_list)
            res = search.execute()
            eq_(res.count, 2)

            now = utc_now()
            for row in res.results:
                assert row['_id'] in ('*****@*****.**', '*****@*****.**')
                date = string_to_datetime(row['_source']['last_sending'])
                eq_(date.year, now.year)
                eq_(date.month, now.month)
                eq_(date.day, now.day)

            # Run crontabber again and verify that all users are updated,
            # and emails are not sent twice
            state = tab.job_state_database['automatic-emails']
            self._wind_clock(state, hours=1)
            tab.job_state_database['automatic-emails'] = state

            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert not information['automatic-emails']['last_error']
            assert information['automatic-emails']['last_success']

            # Verify that users were not sent an email twice
            eq_(_email_sent, [
                '*****@*****.**', '*****@*****.**', '*****@*****.**',
                '*****@*****.**', '*****@*****.**'
            ])
Esempio n. 8
0
    def test_email_cannot_be_sent_twice(self, exacttarget_mock):
        config_manager = self._setup_config_manager(
            restrict_products=['NightlyTrain']
        )
        et_mock = exacttarget_mock.return_value

        # Prepare failures
        _failures = []
        _email_sent = []

        class SomeRandomError(Exception):
            pass

        def trigger_send(template, fields):
            email = fields['EMAIL_ADDRESS_']
            if email == '*****@*****.**' and email not in _failures:
                _failures.append(email)
                raise SomeRandomError('This is an error. ')
            else:
                _email_sent.append(email)

        et_mock.trigger_send = trigger_send

        with config_manager.context() as config:
            tab = crontabber.CronTabber(config)
            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert information['automatic-emails']['last_error']
            self.assertEqual(
                information['automatic-emails']['last_error']['type'],
                str(SomeRandomError)
            )

            # Verify that user's data was updated, but not all of it
            self.assertEqual(_email_sent, ['*****@*****.**', '*****@*****.**'])
            emails_list = (
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**'
            )

            conf = config.crontabber['class-AutomaticEmailsCronApp']
            es = SuperS().es(
                urls=conf.elasticsearch.elasticsearch_urls,
                timeout=conf.elasticsearch.elasticsearch_timeout,
            )
            search = es.indexes(
                conf.elasticsearch.elasticsearch_emails_index
            )
            search = search.doctypes('emails')
            es.get_es().refresh()

            search = search.filter(_id__in=emails_list)
            res = search.execute()
            self.assertEqual(res.count, 2)

            now = utc_now()
            for row in res.results:
                assert row['_id'] in ('*****@*****.**', '*****@*****.**')
                date = string_to_datetime(row['_source']['last_sending'])
                self.assertEqual(date.year, now.year)
                self.assertEqual(date.month, now.month)
                self.assertEqual(date.day, now.day)

            # Run crontabber again and verify that all users are updated,
            # and emails are not sent twice
            state = tab.job_database['automatic-emails']
            self._wind_clock(state, hours=1)
            tab.job_database['automatic-emails'] = state

            tab.run_all()

            information = self._load_structure()
            assert information['automatic-emails']
            assert not information['automatic-emails']['last_error']
            assert information['automatic-emails']['last_success']

            # Verify that users were not sent an email twice
            self.assertEqual(_email_sent, [
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**',
                '*****@*****.**'
            ])