def test_updates_data_as_new_testruns_arrive(self): build = self.create_build('1', datetime=h(10), create_test_run=False) tests_json = """ { "tests/foo": "pass" } """ self.receive_testrun(build.version, self.environment.slug, tests_file=tests_json) ProjectStatus.create_or_update(build) tests_json = """ { "tests/bar": "pass", "tests/baz": "fail", "tests/qux": "none" } """ self.receive_testrun(build.version, self.environment.slug, tests_file=tests_json) test_run2 = build.test_runs.create(environment=self.environment) test_run2.metrics.create(name='v1', suite=self.suite, result=5.0) status = ProjectStatus.create_or_update(build) build.refresh_from_db() status.refresh_from_db() self.assertEqual(status, build.status) self.assertEqual(2, status.tests_pass) self.assertEqual(1, status.tests_fail) self.assertEqual(1, status.tests_skip) self.assertEqual(status.tests_pass, build.status.tests_pass) self.assertEqual(status.tests_fail, build.status.tests_fail) self.assertEqual(status.tests_skip, build.status.tests_skip) self.assertAlmostEqual(5.0, status.metrics_summary) self.assertEqual(status.metrics_summary, build.status.metrics_summary)
def test_send_notification_on_change_only_with_no_changes(self): self.project.notification_strategy = Project.NOTIFY_ON_CHANGE self.project.save() self.project.subscriptions.create(email='*****@*****.**') ProjectStatus.create_or_update(self.build2) send_notification(self.project) self.assertEqual(0, len(mail.outbox))
def test_last_build_comparison(self): # Test that the build that we compare against is truly the last one # time wise. build1 = self.create_build('1', datetime=h(10)) test_run1 = build1.test_runs.first() foo_metadata, _ = SuiteMetadata.objects.get_or_create(suite=self.suite.slug, name='foo', kind='test') bar_metadata, _ = SuiteMetadata.objects.get_or_create(suite=self.suite.slug, name='bar', kind='test') test_run1.tests.create(build=test_run1.build, environment=test_run1.environment, metadata=foo_metadata, suite=self.suite, result=False) test_run1.tests.create(build=test_run1.build, environment=test_run1.environment, metadata=bar_metadata, suite=self.suite, result=False) ProjectStatus.create_or_update(build1) build2 = self.create_build('2', datetime=h(9)) test_run2 = build2.test_runs.first() test_run2.tests.create(build=test_run2.build, environment=test_run2.environment, metadata=foo_metadata, suite=self.suite, result=False) test_run2.tests.create(build=test_run2.build, environment=test_run2.environment, metadata=bar_metadata, suite=self.suite, result=True) ProjectStatus.create_or_update(build2) build3 = self.create_build('3', datetime=h(8)) test_run3 = build3.test_runs.first() test_run3.tests.create(build=test_run3.build, environment=test_run3.environment, metadata=foo_metadata, suite=self.suite, result=True) test_run3.tests.create(build=test_run3.build, environment=test_run3.environment, metadata=bar_metadata, suite=self.suite, result=True) status3 = ProjectStatus.create_or_update(build3) fixes3 = status3.get_fixes() self.assertEqual(len(fixes3['theenvironment']), 1) self.assertEqual(fixes3['theenvironment'][0], 'suite_/foo')
def test_send_notification_only_once(self, diff): self.project.subscriptions.create(email='*****@*****.**') diff.return_value = fake_diff() ProjectStatus.create_or_update(self.build2) send_notification(self.project) send_notification(self.project) self.assertEqual(1, len(mail.outbox))
def test_cache_fixes_update(self): build1 = self.create_build('1', datetime=h(10)) test_run1 = build1.test_runs.first() foo_metadata, _ = SuiteMetadata.objects.get_or_create( suite=self.suite.slug, name='foo', kind='test') test_run1.tests.create(build=test_run1.build, environment=test_run1.environment, metadata=foo_metadata, suite=self.suite, result=False) ProjectStatus.create_or_update(build1) build2 = self.create_build('2', datetime=h(9)) test_run2 = build2.test_runs.first() test_run2.tests.create(build=test_run2.build, environment=test_run2.environment, metadata=foo_metadata, suite=self.suite, result=False) status1 = ProjectStatus.create_or_update(build2) self.assertIsNone(status1.fixes) self.assertIsNone(status1.regressions) build3 = self.create_build('3', datetime=h(8)) test_run3 = build3.test_runs.first() test_run3.tests.create(build=test_run3.build, environment=test_run3.environment, metadata=foo_metadata, suite=self.suite, result=True) status2 = ProjectStatus.create_or_update(build3) self.assertIsNotNone(status2.fixes) self.assertIsNone(status2.regressions)
def test_get_exceeded_thresholds(self): build = self.create_build('1') testrun = build.test_runs.create(environment=self.environment) testrun.metrics.create(name='metric1', suite=self.suite, result=3) testrun.metrics.create(name='metric2', suite=self.suite, result=8) testrun.metrics.create(name='metric3', suite=self.suite, result=5) build_a = self.create_build('2') testrun_a = build_a.test_runs.create(environment=self.environment_a) testrun_a.metrics.create(name='metric4', suite=self.suite_a, result=3) testrun_a.metrics.create(name='metric5', suite=self.suite_a, result=2) testrun_a.metrics.create(name='metric6', suite=self.suite_a, result=7) status = ProjectStatus.create_or_update(build) MetricThreshold.objects.create(environment=self.environment, name='suite_/metric2', value=4, is_higher_better=False) thresholds = status.get_exceeded_thresholds() self.assertEqual(len(thresholds), 1) self.assertEqual(thresholds[0][1].name, 'metric2') self.assertEqual(thresholds[0][1].result, 8) status_a = ProjectStatus.create_or_update(build_a) MetricThreshold.objects.create(environment=self.environment_a, name='suite_a/metric6', value=4, is_higher_better=True) thresholds = status_a.get_exceeded_thresholds() self.assertEqual(len(thresholds), 0)
def test_last_build_not_finished(self): self.create_build('0', datetime=h(10)) ProjectStatus.create(self.project) b1 = self.create_build('1', datetime=h(5)) self.create_build('2', datetime=h(4), create_test_run=False) status = ProjectStatus.create(self.project) self.assertEqual(b1, status.build)
def test_send_notification_on_change_only(self, diff): diff.return_value = fake_diff() self.project.notification_strategy = Project.NOTIFY_ON_CHANGE self.project.save() self.project.subscriptions.create(email='*****@*****.**') ProjectStatus.create_or_update(self.build2) send_notification(self.project) self.assertEqual(1, len(mail.outbox))
def test_status_of_second_build(self): self.create_build('1') status1 = ProjectStatus.create(self.project) build2 = self.create_build('2') status2 = ProjectStatus.create(self.project) self.assertEqual(status1, status2.previous) self.assertEqual(build2, status2.build)
def test_status_of_second_build(self): build = self.create_build('1') status1 = ProjectStatus.create_or_update(build) build2 = self.create_build('2') status2 = ProjectStatus.create_or_update(build2) self.assertEqual(status1, status2.get_previous()) self.assertEqual(build2, status2.build)
def test_status_with_multiple_builds(self): self.create_build('1', datetime=h(10)) ProjectStatus.create(self.project) b1 = self.create_build('2', datetime=h(5)) b2 = self.create_build('3', datetime=h(4)) status = ProjectStatus.create(self.project) self.assertEqual([b1, b2], list(status.builds))
def test_send_plain_text_only(self): self.project.subscriptions.create(email='*****@*****.**') self.project.html_mail = False self.project.save() ProjectStatus.create_or_update(self.build2) send_notification(self.project) msg = mail.outbox[0] self.assertEqual(0, len(msg.alternatives))
def setUp(self): t0 = timezone.now() - relativedelta(hours=3) t = timezone.now() - relativedelta(hours=3) self.group = Group.objects.create(slug='mygroup') self.project = self.group.projects.create(slug='myproject') self.project.builds.create(version='1', datetime=t0) ProjectStatus.create(self.project) self.project.builds.create(version='2', datetime=t)
def test_send_all_pending_notifications(self): self.project.subscriptions.create(email='*****@*****.**') ProjectStatus.create_or_update(self.build2) t = timezone.now() - relativedelta(hours=2.5) build3 = self.project.builds.create(version='3', datetime=t) ProjectStatus.create_or_update(build3) send_notification(self.project) self.assertEqual(2, len(mail.outbox))
def test_cache_test_run_counts_on_update(self): build = self.create_build('1', create_test_run=False) ProjectStatus.create_or_update(build) build.test_runs.create(environment=self.environment, completed=True) build.test_runs.create(environment=self.environment, completed=False) status = ProjectStatus.create_or_update(build) self.assertEqual(2, status.test_runs_total) self.assertEqual(1, status.test_runs_completed) self.assertEqual(1, status.test_runs_incomplete)
def test_updates_last_updated(self): build = self.create_build('1', datetime=h(10)) test_run1 = build.test_runs.first() test_run1.tests.create(name='foo', suite=self.suite, result=True) status = ProjectStatus.create_or_update(build) old_date = status.last_updated build.test_runs.create(environment=self.environment) status = ProjectStatus.create_or_update(build) self.assertNotEqual(status.last_updated, old_date)
def test_updates_last_updated(self): build = self.create_build('1', datetime=h(10)) test_run1 = build.test_runs.first() metadata, _ = SuiteMetadata.objects.get_or_create(suite=self.suite.slug, name='foo', kind='test') test_run1.tests.create(build=test_run1.build, environment=test_run1.environment, metadata=metadata, suite=self.suite, result=True) status = ProjectStatus.create_or_update(build) old_date = status.last_updated build.test_runs.create(environment=self.environment) status = ProjectStatus.create_or_update(build) self.assertNotEqual(status.last_updated, old_date)
def test_previous_must_be_from_the_same_project(self): previous_build = self.create_build('1', datetime=h(10)) previous = ProjectStatus.create_or_update(previous_build) other_project = self.group.projects.create(slug='other_project') other_env = other_project.environments.create(slug='other_env') other_build = other_project.builds.create(version='1', datetime=h(5)) other_build.test_runs.create(environment=other_env) ProjectStatus.create_or_update(other_build) build = self.create_build('2', datetime=h(0)) status = ProjectStatus.create_or_update(build) self.assertEqual(previous, status.get_previous())
def test_cache_fixes(self): build1 = self.create_build('1', datetime=h(10)) test_run1 = build1.test_runs.first() test_run1.tests.create(name='foo', suite=self.suite, result=False) ProjectStatus.create_or_update(build1) build2 = self.create_build('2', datetime=h(9)) test_run2 = build2.test_runs.first() test_run2.tests.create(name='foo', suite=self.suite, result=True) status = ProjectStatus.create_or_update(build2) self.assertIsNotNone(status.fixes) self.assertIsNone(status.regressions)
def test_delegates_diff_to_test_comparison_object(self, diff): the_diff = {} diff.return_value = the_diff group = Group.objects.create(slug='mygroup') project = group.projects.create(slug='myproject') build1 = project.builds.create(version='1') ProjectStatus.create_or_update(build1) build2 = project.builds.create(version='2') status = ProjectStatus.create_or_update(build2) notification = Notification(status) self.assertIs(the_diff, notification.diff)
def test_send_notification_for_all_builds(self, diff): self.project.subscriptions.create(email='*****@*****.**') diff.return_value = fake_diff() status1 = ProjectStatus.create_or_update(self.build2) send_status_notification(status1) self.assertEqual(1, len(mail.outbox)) t = timezone.now() - relativedelta(hours=2.5) build = self.project.builds.create(version='3', datetime=t) status2 = ProjectStatus.create_or_update(build) send_status_notification(status2) self.assertEqual(2, len(mail.outbox))
def test_status_with_multiple_builds(self): self.create_build('1', datetime=timezone.now() - relativedelta(hours=10)) ProjectStatus.create(self.project) b1 = self.create_build('2', datetime=timezone.now() - relativedelta(hours=5)) b2 = self.create_build('3', datetime=timezone.now() - relativedelta(hours=4)) status = ProjectStatus.create(self.project) self.assertEqual([b1, b2], list(status.builds))
def handle(self, *args, **options): self.options = options builds = Build.objects.filter(datetime__gt=timezone.make_aware( self.options['date_start']), status__finished=True) total = builds.count() for index, build in enumerate(builds): ProjectStatus.create_or_update(build) if index % 100 == 0: logger.info('Progress: {1:>2}%[{0:10}]'.format( '#' * int((index + 1) * 10 / total), int((index + 1) * 100 / total)), end='')
def test_send_notification_for_all_builds(self, diff): self.project.subscriptions.create(email='*****@*****.**') diff.return_value = fake_diff() ProjectStatus.create_or_update(self.build2) send_notification(self.project) self.assertEqual(1, len(mail.outbox)) t = timezone.now() - relativedelta(hours=2.5) build = self.project.builds.create(version='3', datetime=t) ProjectStatus.create_or_update(build) # project.status is cached, get a new instance project = Project.objects.get(pk=self.project.id) send_notification(project) self.assertEqual(2, len(mail.outbox))
def test_no_recipients_no_email_mark_as_notified(self, diff): diff.return_value = fake_diff() status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) self.assertEqual(0, len(mail.outbox)) status.refresh_from_db() self.assertTrue(status.notified)
def setUp(self): t0 = timezone.now() - relativedelta(hours=3) t = timezone.now() - relativedelta(hours=2.75) self.group = Group.objects.create(slug='mygroup') self.project = self.group.projects.create(slug='myproject') self.build1 = self.project.builds.create(version='1', datetime=t0) status = ProjectStatus.create_or_update(self.build1) status.notified = True status.save() self.build2 = self.project.builds.create(version='2', datetime=t) self.project.subscriptions.create(email='*****@*****.**') self.project.admin_subscriptions.create(email='*****@*****.**') self.project.moderate_notifications = True self.project.save() self.status = ProjectStatus.create_or_update(self.build2)
def test_zero_expected_test_runs(self): self.project.environments.create(slug='other_env', expected_test_runs=0) build = self.create_build('1') status = ProjectStatus.create_or_update(build) self.assertTrue(status.finished)
def test_send_notification_on_regression_only_with_no_regressions(self): self.project.subscriptions.create( email='*****@*****.**', notification_strategy=Subscription.NOTIFY_ON_REGRESSION) status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) self.assertEqual(0, len(mail.outbox))
def __call__(testrun): projectstatus = ProjectStatus.create_or_update(testrun.build) if projectstatus.finished: dispatch_callbacks_on_build_finished(testrun.build) maybe_notify_project_status.delay(projectstatus.id)
def test_send_notification_on_change_only(self, diff): diff.return_value = fake_diff() self.project.subscriptions.create( email='*****@*****.**', notification_strategy=Subscription.NOTIFY_ON_CHANGE) status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) self.assertEqual(1, len(mail.outbox))
def test_maybe_notify_project_status_schedule_timeout_not_requested(self, apply_async): build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) maybe_notify_project_status(status.id) apply_async.assert_not_called()
def test_maybe_notify_project_status(self, send_status_notification): build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) maybe_notify_project_status(status.id) send_status_notification.assert_called_with(status)
def test_maybe_notify_project_status_notify_patch_build_finished(self, notify_patch_build_finished): build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) maybe_notify_project_status(status.id) notify_patch_build_finished.delay.assert_called_with(build.id)
def test_send_notification_on_regression_only(self, regressions): regressions.return_value = OrderedDict({'key': 'value'}) self.project.subscriptions.create( email='*****@*****.**', notification_strategy=Subscription.NOTIFY_ON_REGRESSION) status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) self.assertEqual(1, len(mail.outbox))
def test_metrics_summary(self): build = self.create_build('1', datetime=h(10)) test_run = build.test_runs.first() test_run.metrics.create(name='foo', suite=self.suite, result=2) test_run.metrics.create(name='bar', suite=self.suite, result=2) status = ProjectStatus.create_or_update(build) self.assertEqual(2.0, status.metrics_summary)
def test_send_plain_text_only(self): self.project.subscriptions.create(email='*****@*****.**') self.project.html_mail = False self.project.save() status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) msg = mail.outbox[0] self.assertEqual(0, len(msg.alternatives))
def test_previous_must_be_finished(self): self.environment.expected_test_runs = 2 self.environment.save() # finished build1 = self.create_build('1', datetime=h(10), create_test_run=False) build1.test_runs.create(environment=self.environment) build1.test_runs.create(environment=self.environment) status1 = ProjectStatus.create_or_update(build1) # not finished build2 = self.create_build('2', datetime=h(5), create_test_run=False) ProjectStatus.create_or_update(build2) # current build build = self.create_build('3', datetime=h(0), create_test_run=False) status = ProjectStatus.create_or_update(build) self.assertEqual(status1, status.get_previous())
def test_maybe_notify_project_status_schedule_timeout(self, apply_async): self.project1.notification_timeout = 3600 # 1 hour self.project1.save() build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) maybe_notify_project_status(status.id) apply_async.assert_called_with(args=[status.id], countdown=3600)
def test_notification_timeout(self, send_status_notification): self.project1.notification_timeout = 3600 # 1 hour self.project1.save() build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) notification_timeout(status.id) send_status_notification.assert_called_with(status)
def test_subject_from_custom_template(self): template = EmailTemplate.objects.create(subject='lalala', plain_text='foo', html='bar') self.project.use_custom_email_template = True self.project.custom_email_template = template self.project.save() status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) msg = mail.outbox[0] self.assertEqual('lalala', msg.subject)
def handle(self, *args, **options): self.options = options builds = Build.objects.filter( datetime__range=( timezone.make_aware(self.options['date_start']), timezone.make_aware(self.options['date_end'])), status__finished=True ) total = builds.count() logger.info("Updating ProjectStatus objects from builds...") logger.info("Total build count in selected date range: %s" % total) for index, build in enumerate(builds.iterator()): ProjectStatus.create_or_update(build) if index % 100 == 0: logger.info('Progress: {1:>2}%[{0:10}]'.format( '#' * int((index + 1) * 10 / total), int((index + 1) * 100 / total))) logger.info('Progress: {1:>2}%[{0:10}]'.format('#' * 10, 100))
def test_notification_timeout_only_once(self, send_status_notification): self.project1.notification_timeout = 3600 # 1 hour self.project1.save() build = self.project1.builds.create(datetime=timezone.now()) environment = self.project1.environments.create(slug='env') build.test_runs.create(environment=environment) status = ProjectStatus.create_or_update(build) notification_timeout(status.id) notification_timeout(status.id) self.assertEqual(1, len(send_status_notification.call_args_list))
def test_test_summary(self): build = self.create_build('1', datetime=h(10)) test_run = build.test_runs.first() test_run.tests.create(name='foo', suite=self.suite, result=True) test_run.tests.create(name='bar', suite=self.suite, result=False) test_run.tests.create(name='baz', suite=self.suite, result=None) status = ProjectStatus.create_or_update(build) self.assertEqual(1, status.tests_pass) self.assertEqual(1, status.tests_fail) self.assertEqual(1, status.tests_skip) self.assertEqual(3, status.tests_total)
def test_updates_data_as_new_testruns_arrive(self): build = self.create_build('1', datetime=h(10)) test_run1 = build.test_runs.first() test_run1.tests.create(name='foo', suite=self.suite, result=True) ProjectStatus.create_or_update(build) test_run2 = build.test_runs.create(environment=self.environment) test_run2.tests.create(name='bar', suite=self.suite, result=True) test_run2.tests.create(name='baz', suite=self.suite, result=False) test_run2.tests.create(name='qux', suite=self.suite, result=None) test_run2.metrics.create(name='v1', suite=self.suite, result=5.0) status = ProjectStatus.create_or_update(build) self.assertEqual(2, status.tests_pass) self.assertEqual(1, status.tests_fail) self.assertEqual(1, status.tests_skip) self.assertEqual(status.tests_pass, build.status.tests_pass) self.assertEqual(status.tests_fail, build.status.tests_fail) self.assertEqual(status.tests_skip, build.status.tests_skip) self.assertAlmostEqual(5.0, status.metrics_summary) self.assertEqual(status.metrics_summary, build.status.metrics_summary)
def test_dont_html_escape_metadata_for_plain_text(self): self.project.subscriptions.create(email='*****@*****.**') self.project.html_mail = False self.project.save() env = self.project.environments.create(slug='myenv') self.build2.test_runs.create(environment=env, metadata_file='{"foo": "foo\'bar"}') status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) msg = mail.outbox[0] self.assertIn("foo'bar", msg.body)
def setUp(self): t0 = timezone.now() - relativedelta(hours=3) t = timezone.now() - relativedelta(hours=2.75) self.group = Group.objects.create(slug='mygroup') self.user = User.objects.create(username='******', email="*****@*****.**") self.project = self.group.projects.create(slug='myproject') self.build1 = self.project.builds.create(version='1', datetime=t0) status = ProjectStatus.create_or_update(self.build1) status.notified = True status.save() self.build2 = self.project.builds.create(version='2', datetime=t)
def __call__(testrun): projectstatus = ProjectStatus.create_or_update(testrun.build) try: maybe_notify_project_status.delay(projectstatus.id) except OSError as e: # can't request background task for some reason; log the error # and continue. # # This will happen as "OSError: [Errno 111] Connection refused" # in development environments without a running AMQP server, # but also on production setups that are not running the # background job processes because they don't need email # notifications or CI integration logger.error("Cannot schedule notification: " + str(e) + "\n" + traceback.format_exc())
def test_last_build_comparison(self): # Test that the build that we compare agains is truly the last one # time wise. build1 = self.create_build('1', datetime=h(10)) test_run1 = build1.test_runs.first() test_run1.tests.create(name='foo', suite=self.suite, result=False) test_run1.tests.create(name='bar', suite=self.suite, result=False) ProjectStatus.create_or_update(build1) build2 = self.create_build('2', datetime=h(9)) test_run2 = build2.test_runs.first() test_run2.tests.create(name='foo', suite=self.suite, result=False) test_run2.tests.create(name='bar', suite=self.suite, result=True) ProjectStatus.create_or_update(build2) build3 = self.create_build('3', datetime=h(8)) test_run3 = build3.test_runs.first() test_run3.tests.create(name='foo', suite=self.suite, result=True) test_run3.tests.create(name='bar', suite=self.suite, result=True) status3 = ProjectStatus.create_or_update(build3) fixes3 = yaml.load(status3.fixes) self.assertEqual(len(fixes3['theenvironment']), 1) self.assertEqual(fixes3['theenvironment'][0], 'foo')
def test_custom_template(self): template = EmailTemplate.objects.create(plain_text='foo', html='bar') self.project.use_custom_email_template = True self.project.custom_email_template = template self.project.save() status = ProjectStatus.create_or_update(self.build2) send_status_notification(status) msg = mail.outbox[0] txt = msg.body html = msg.alternatives[0][0] self.assertEqual('foo', txt) self.assertEqual('bar', html)