def test_delete_old_sessions_with_no_reference(self): """ Delete sessions that have snapshots (but no reference, with a ttl > 0, which are older than the ttl """ s1 = TestSession(sessionId="1234", date=timezone.now() - datetime.timedelta(days=4), browser="firefox", environment=TestEnvironment.objects.get(pk=1), version=Version.objects.get(pk=1), ttl=datetime.timedelta(days=0)) s1.save() step = TestStep(name="step1") step.save() tc1 = TestCase(name="case1", application=Application.objects.get(pk=1)) tc1.save() tcs1 = TestCaseInSession(testCase=tc1, session=s1) tcs1.save() tcs1.testSteps.set([step]) tcs1.save() tsr1 = StepResult(step=step, testCase=tcs1, result=True) tsr1.save() # add a snapshot without reference => its a reference itself sn0 = Snapshot(stepResult=StepResult.objects.get(pk=1), image=None, refSnapshot=None, pixelsDiff=None) sn0.save() sn1 = Snapshot(stepResult=tsr1, image=None, refSnapshot=sn0, pixelsDiff=None) sn1.save() s1.ttl = datetime.timedelta(days=3) s1.save() # s1 should have been deleted self.assertRaises(TestSession.DoesNotExist, TestSession.objects.get, pk=s1.id)
def test_is_ko_with_snapshots_ko_and_result_ok(self): """ Step is KO when at least one snapshot is KO even if step result is OK """ tcs = TestCaseInSession.objects.get(pk=5) step = TestStep.objects.get(pk=2) step_result = StepResult(step=step, testCase=tcs, result=True) # step without snapshots step_result.save() # one snapshot with comparison error snapshot1 = Snapshot(stepResult=step_result, image=None, refSnapshot=None, pixelsDiff=b"12345", tooManyDiffs=True) snapshot1.save() # one snapshot without comparison error snapshot2 = Snapshot(stepResult=step_result, image=None, refSnapshot=None, pixelsDiff=None, tooManyDiffs=False) snapshot2.save() # Step is KO because one snapshot has comparison error self.assertFalse(step.isOkWithSnapshots(tcs))
def test_is_ko_without_snapshots_and_result_ko(self): """ Step is OK when no snapshot is present and step result is OK """ tcs = TestCaseInSession.objects.get(pk=5) step = TestStep.objects.get(pk=2) step_result = StepResult(step=step, testCase=tcs, result=False) # step without snapshots step_result.save() self.assertFalse(step.isOkWithSnapshots(tcs))
def test_post_snapshot_existing_ref_in_previous_version(self): """ Check that we search for a reference in a previous version if none is found in the current one """ # same as self.testCase in a greater version session3 = TestSession(sessionId="8890", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=2), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) session3.save() tcs3 = TestCaseInSession(testCase=self.testCase, session=session3) tcs3.save() tcs3.testSteps.set([TestStep.objects.get(id=1)]) tcs3.save() sr3 = StepResult(step=TestStep.objects.get(id=1), testCase=tcs3, result=True) sr3.save() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': sr3.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=tcs3, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_2, "the uploaded snapshot should be recorded") self.assertEqual(uploaded_snapshot_2.refSnapshot, uploaded_snapshot_1)
def setUp(self): self.client = Client() authenticate_test_client_for_web_view(self.client) # prepare data self.testCase = TestCase.objects.get(id=1) self.initialRefSnapshot = Snapshot.objects.get(id=1) self.step1 = TestStep.objects.get(id=1) self.session1 = TestSession( sessionId="1237", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session1.save() self.tcs1 = TestCaseInSession(testCase=self.testCase, session=self.session1) self.tcs1.save() self.sr1 = StepResult(step=self.step1, testCase=self.tcs1, result=True) self.sr1.save() self.session_same_env = TestSession( sessionId="1238", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_same_env.save() self.tcs_same_env = TestCaseInSession(testCase=self.testCase, session=self.session_same_env) self.tcs_same_env.save() self.step_result_same_env = StepResult(step=self.step1, testCase=self.tcs_same_env, result=True) self.step_result_same_env.save() # session with other env (AUT instead of DEV), other characteristics remain the same as session1 self.session_other_env = TestSession.objects.get(pk=10) self.tcs_other_env = TestCaseInSession.objects.get(pk=9) self.step_result_other_env = StepResult.objects.get(pk=11) # session with other browser (chrome instead of firefox), other characteristics remain the same as session1 self.session_other_browser = TestSession.objects.get(pk=11) self.tcs_other_browser = TestCaseInSession.objects.get(pk=10) self.step_result_other_browser = StepResult.objects.get(pk=12)
def test_is_ok_with_one_step_ko(self): tcs = TestCaseInSession.objects.get(pk=5) s1 = TestStep.objects.get(pk=2) s2 = TestStep.objects.get(pk=3) sr1 = StepResult(step=s1, testCase=tcs, result=False) sr1.save() sr2 = StepResult(step=s2, testCase=tcs, result=True) sr2.save() self.assertFalse(tcs.isOkWithResult())
def setUp(self): super().setUp() self.app = Application(name="test") self.app.save() self.v1 = Version(application=self.app, name='1.0') self.v1.save() self.v2 = Version(application=self.app, name='2.0') self.v2.save() env = TestEnvironment(name='DEV') env.save() self.session1 = TestSession(sessionId="1234", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v1, ttl=datetime.timedelta(0)) self.session1.save() self.session_same_env = TestSession(sessionId="1235", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v1, ttl=datetime.timedelta(0)) self.session_same_env.save() self.session3 = TestSession(sessionId="1236", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v2, ttl=datetime.timedelta(0)) self.session3.save() self.session4 = TestSession(sessionId="1237", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v2, ttl=datetime.timedelta(0)) self.session4.save() self.step = TestStep(name="step1") self.step.save() self.tc1 = TestCase(name="case1", application=self.app) self.tc1.save() self.tcs1 = TestCaseInSession(testCase=self.tc1, session=self.session1) self.tcs1.save() self.tcs_same_env = TestCaseInSession(testCase=self.tc1, session=self.session_same_env) self.tcs_same_env.save() self.tcs3 = TestCaseInSession(testCase=self.tc1, session=self.session3) self.tcs3.save() self.tcs4 = TestCaseInSession(testCase=self.tc1, session=self.session4) self.tcs4.save() self.tcs1.testSteps.set([self.step]) self.tcs1.save() self.tcs_same_env.testSteps.set([self.step]) self.tcs_same_env.save() self.tsr1 = StepResult(step=self.step, testCase=self.tcs1, result=True) self.tsr1.save() self.tsr2 = StepResult(step=self.step, testCase=self.tcs_same_env, result=True) self.tsr2.save() self.tsr3 = StepResult(step=self.step, testCase=self.tcs3, result=True) self.tsr3.save() self.tsr4 = StepResult(step=self.step, testCase=self.tcs4, result=True) self.tsr4.save()
def clean(self): super().clean() try: self.cleaned_data['stepName'] self.cleaned_data['testCaseName'] self.cleaned_data['versionId'] self.cleaned_data['environmentId'] self.cleaned_data['browser'] except KeyError as e: raise forms.ValidationError( "stepName, testCaseName, version, environment, browser must be specified" ) # create the StepResult object from provided data version = Version.objects.get(id=self.cleaned_data['versionId']) environment = TestEnvironment.objects.get( id=self.cleaned_data['environmentId']) test_case = TestCase(name=self.cleaned_data['testCaseName'], application=version.application) test_session = TestSession(sessionId='123', version=version, browser=self.cleaned_data['browser'], environment=environment, compareSnapshot=True, ttl=datetime.timedelta(days=0)) step = TestStep.objects.get(name=self.cleaned_data['stepName']) test_case_in_session = TestCaseInSession(testCase=test_case, session=test_session) self.cleaned_data['stepResult'] = StepResult( step=step, testCase=test_case_in_session, result=True) self.cleaned_data['storeSnapshot'] = False if self.cleaned_data['diffTolerance'] == None: self.cleaned_data['diffTolerance'] = 0.0 if self.cleaned_data['compare'] not in ['true', 'false']: self.cleaned_data['compare'] = 'true'
class TestViews(SnapshotTestCase): fixtures = ['snapshotServer.yaml'] dataDir = 'snapshotServer/tests/data/' media_dir = settings.MEDIA_ROOT + os.sep + 'documents' def setUp(self): self.client = Client() authenticate_test_client_for_web_view(self.client) # prepare data self.testCase = TestCase.objects.get(id=1) self.initialRefSnapshot = Snapshot.objects.get(id=1) self.step1 = TestStep.objects.get(id=1) self.session1 = TestSession( sessionId="1237", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session1.save() self.tcs1 = TestCaseInSession(testCase=self.testCase, session=self.session1) self.tcs1.save() self.sr1 = StepResult(step=self.step1, testCase=self.tcs1, result=True) self.sr1.save() self.session_same_env = TestSession( sessionId="1238", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_same_env.save() self.tcs_same_env = TestCaseInSession(testCase=self.testCase, session=self.session_same_env) self.tcs_same_env.save() self.step_result_same_env = StepResult(step=self.step1, testCase=self.tcs_same_env, result=True) self.step_result_same_env.save() # session with other env (AUT instead of DEV), other characteristics remain the same as session1 self.session_other_env = TestSession.objects.get(pk=10) self.tcs_other_env = TestCaseInSession.objects.get(pk=9) self.step_result_other_env = StepResult.objects.get(pk=11) # session with other browser (chrome instead of firefox), other characteristics remain the same as session1 self.session_other_browser = TestSession.objects.get(pk=11) self.tcs_other_browser = TestCaseInSession.objects.get(pk=10) self.step_result_other_browser = StepResult.objects.get(pk=12) def tearDown(self): """ Remove generated files """ super().tearDown() for f in os.listdir(self.media_dir): if f.startswith('img_'): os.remove(self.media_dir + os.sep + f)
def test_delete_old_sessions_with_reference(self): """ Do not delete sessions that have reference snapshots, with a ttl > 0, which are older than the ttl """ s1 = TestSession(sessionId="1234", date=timezone.now() - datetime.timedelta(days=4), browser="firefox", environment=TestEnvironment.objects.get(pk=1), version=Version.objects.get(pk=1), ttl=datetime.timedelta(days=0)) s1.save() step1 = TestStep(name="step1") step1.save() step2 = TestStep(name="step2") step2.save() tc1 = TestCase(name="case1", application=Application.objects.get(pk=1)) tc1.save() tcs1 = TestCaseInSession(testCase=tc1, session=s1) tcs1.save() tcs1.testSteps.set([step1]) tcs1.save() tsr1 = StepResult(step=step1, testCase=tcs1, result=True) tsr1.save() tsr2 = StepResult(step=step2, testCase=tcs1, result=True) tsr2.save() # add a snapshot without reference => its a reference itself sn1 = Snapshot(stepResult=tsr1, image=None, refSnapshot=None, pixelsDiff=None) sn1.save() sn2 = Snapshot(stepResult=tsr2, image=None, refSnapshot=sn1, pixelsDiff=None) sn2.save() s1.ttl = datetime.timedelta(days=3) s1.save() # old session without reference s2 = TestSession(sessionId="1235", date=timezone.now() - datetime.timedelta(days=4), browser="firefox", environment=TestEnvironment.objects.get(pk=1), version=Version.objects.get(pk=1), ttl=datetime.timedelta(days=0)) s2.save() tcs2 = TestCaseInSession(testCase=tc1, session=s2) tcs2.save() tcs2.testSteps.set([step1]) tcs2.save() tsr2 = StepResult(step=step1, testCase=tcs2, result=True) tsr2.save() # add a snapshot with reference sn2 = Snapshot(stepResult=tsr2, image=None, refSnapshot=sn1, pixelsDiff=None) sn2.save() s2.ttl = datetime.timedelta(days=3) s2.save() # session1 should not be deleted TestSession.objects.get(pk=s1.id) # session2 should be deleted self.assertRaises(TestSession.DoesNotExist, TestSession.objects.get, pk=s2.id)
def setUp(self): authenticate_test_client_for_api(self.client) # test in a version self.testCase = TestCase(name='test upload', application=Application.objects.get(id=1)) self.testCase.save() self.step1 = TestStep.objects.get(id=1) self.session1 = TestSession( sessionId="8888", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session1.save() self.tcs1 = TestCaseInSession(testCase=self.testCase, session=self.session1) self.tcs1.save() self.sr1 = StepResult(step=self.step1, testCase=self.tcs1, result=True) self.sr1.save() self.session_same_env = TestSession( sessionId="8889", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_same_env.save() self.tcs_same_env = TestCaseInSession(testCase=self.testCase, session=self.session_same_env) self.tcs_same_env.save() self.step_result_same_env = StepResult(step=self.step1, testCase=self.tcs_same_env, result=True) self.step_result_same_env.save() self.session_other_env = TestSession( sessionId="8890", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=2), ttl=datetime.timedelta(0)) self.session_other_env.save() self.tcs_other_env = TestCaseInSession(testCase=self.testCase, session=self.session_other_env) self.tcs_other_env.save() self.step_result_other_env = StepResult(step=self.step1, testCase=self.tcs_other_env, result=True) self.step_result_other_env.save() self.session_other_browser = TestSession( sessionId="8891", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="chrome", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_other_browser.save() self.tcs_other_browser = TestCaseInSession( testCase=self.testCase, session=self.session_other_browser) self.tcs_other_browser.save() self.step_result_other_browser = StepResult( step=self.step1, testCase=self.tcs_other_browser, result=True) self.step_result_other_browser.save()
class TestFileUploadView(APITestCase): fixtures = ['snapshotServer.yaml'] media_dir = settings.MEDIA_ROOT + os.sep + 'documents' def setUp(self): authenticate_test_client_for_api(self.client) # test in a version self.testCase = TestCase(name='test upload', application=Application.objects.get(id=1)) self.testCase.save() self.step1 = TestStep.objects.get(id=1) self.session1 = TestSession( sessionId="8888", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session1.save() self.tcs1 = TestCaseInSession(testCase=self.testCase, session=self.session1) self.tcs1.save() self.sr1 = StepResult(step=self.step1, testCase=self.tcs1, result=True) self.sr1.save() self.session_same_env = TestSession( sessionId="8889", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_same_env.save() self.tcs_same_env = TestCaseInSession(testCase=self.testCase, session=self.session_same_env) self.tcs_same_env.save() self.step_result_same_env = StepResult(step=self.step1, testCase=self.tcs_same_env, result=True) self.step_result_same_env.save() self.session_other_env = TestSession( sessionId="8890", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=2), ttl=datetime.timedelta(0)) self.session_other_env.save() self.tcs_other_env = TestCaseInSession(testCase=self.testCase, session=self.session_other_env) self.tcs_other_env.save() self.step_result_other_env = StepResult(step=self.step1, testCase=self.tcs_other_env, result=True) self.step_result_other_env.save() self.session_other_browser = TestSession( sessionId="8891", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="chrome", version=Version.objects.get(pk=1), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) self.session_other_browser.save() self.tcs_other_browser = TestCaseInSession( testCase=self.testCase, session=self.session_other_browser) self.tcs_other_browser.save() self.step_result_other_browser = StepResult( step=self.step1, testCase=self.tcs_other_browser, result=True) self.step_result_other_browser.save() def tearDown(self): """ Remove generated files """ super().tearDown() for f in os.listdir(self.media_dir): if f.startswith('engie'): os.remove(self.media_dir + os.sep + f) def test_post_snapshot_no_ref(self): """ Check a reference is created when non is found """ with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) # check returned data: with no ref, no computing error should be raised, but snapshot is considered as computed data = json.loads(response.content.decode('UTF-8'), encoding='UTF-8') self.assertIsNotNone(data['id']) # ID has been provided self.assertTrue(data['computed']) self.assertEqual(data['computingError'], '') self.assertEqual(data['diffPixelPercentage'], 0.0) self.assertFalse(data['tooManyDiffs']) uploaded_snapshot = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot, "the uploaded snapshot should be recorded") self.assertTrue(uploaded_snapshot.computed) self.assertEqual(uploaded_snapshot.diffTolerance, 0.0) def test_post_snapshot_no_ref_with_threshold(self): """ Check a reference is created when non is found """ with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true', 'diffTolerance': 1.5 }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot, "the uploaded snapshot should be recorded") self.assertTrue(uploaded_snapshot.computed) self.assertEqual(uploaded_snapshot.diffTolerance, 1.5) def test_post_snapshot_existing_ref(self): """ Check we find the reference snapshot when it exists in the same version / same name """ with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.step_result_same_env.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=self.tcs_same_env, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_2, "the uploaded snapshot should be recorded") self.assertEqual(uploaded_snapshot_2.refSnapshot, uploaded_snapshot_1) # both snapshots are marked as computed as they have been uploaded self.assertTrue(uploaded_snapshot_1.computed) self.assertTrue(uploaded_snapshot_2.computed) def test_post_snapshot_multiple_existing_ref(self): """ issue #61: Check that when multiple references exist for the same version / name / env / ..., we take the last one. """ # upload first ref with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() # upload second snapshot and make it a reference with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() uploaded_snapshot_2.refSnapshot = None uploaded_snapshot_2.save() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.step_result_same_env.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_3 = Snapshot.objects.filter( stepResult__testCase=self.tcs_same_env, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_3, "the uploaded snapshot should be recorded") self.assertEqual( uploaded_snapshot_3.refSnapshot, uploaded_snapshot_2, "last snapshot should take the most recent reference snapshot available" ) def test_post_snapshot_existing_ref_other_env(self): """ Check we cannot find the reference snapshot when it exists in the same version / same browser / same name but for a different environment """ with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.step_result_other_env.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=self.tcs_other_env, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_2, "the uploaded snapshot should be recorded") # the uploaded snapshot should not have been associated to 'uploaded_snapshot_1' as environment is different self.assertIsNone(uploaded_snapshot_2.refSnapshot) def test_post_snapshot_existing_ref_other_browser(self): """ Check we cannot find the reference snapshot when it exists in the same version / same environment / same name but for a different browser """ with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.step_result_other_browser.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=self.tcs_other_browser, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_2, "the uploaded snapshot should be recorded") # the uploaded snapshot should not have been associated to 'uploaded_snapshot_1' as browser is different self.assertIsNone(uploaded_snapshot_2.refSnapshot) def test_post_snapshot_existing_ref_in_previous_version(self): """ Check that we search for a reference in a previous version if none is found in the current one """ # same as self.testCase in a greater version session3 = TestSession(sessionId="8890", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", version=Version.objects.get(pk=2), environment=TestEnvironment.objects.get(id=1), ttl=datetime.timedelta(0)) session3.save() tcs3 = TestCaseInSession(testCase=self.testCase, session=session3) tcs3.save() tcs3.testSteps.set([TestStep.objects.get(id=1)]) tcs3.save() sr3 = StepResult(step=TestStep.objects.get(id=1), testCase=tcs3, result=True) sr3.save() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'name': 'img', 'compare': 'true' }) uploaded_snapshot_1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': sr3.id, 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual( response.status_code, 201, 'status code should be 201: ' + str(response.content)) uploaded_snapshot_2 = Snapshot.objects.filter( stepResult__testCase=tcs3, stepResult__step__id=1).last() self.assertIsNotNone(uploaded_snapshot_2, "the uploaded snapshot should be recorded") self.assertEqual(uploaded_snapshot_2.refSnapshot, uploaded_snapshot_1) def test_post_snapshot_no_store_picture_parameter(self): """ Check that uploaded picture is not stored when using the "put" method """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test1', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 201, 'status code should be 201') data = json.loads(response.content.decode('UTF-8'), encoding='UTF-8') self.assertIsNone( data['id'] ) # no ID provided, snapshot should not be saved in database self.assertTrue(data['computed']) self.assertEqual(data['computingError'], '') self.assertEqual(data['diffPixelPercentage'], 0.0) self.assertFalse(data['tooManyDiffs']) uploaded_snapshot = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() self.assertIsNone(uploaded_snapshot, "the uploaded snapshot should not be recorded") def test_post_snapshot_with_comparison_no_store_picture_parameter(self): """ Check that uploaded picture is not stored when using the "put" method We expect to get a comparison result """ with open('snapshotServer/tests/data/Ibis_Mulhouse.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'image': fp, 'stepResult': self.sr1.id, 'name': 'img', 'compare': 'true' }) self.assertEqual(response.status_code, 201, 'status code should be 201') uploaded_snapshot1 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() with open('snapshotServer/tests/data/Ibis_Mulhouse_diff.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test upload', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 201, 'status code should be 201') data = json.loads(response.content.decode('UTF-8'), encoding='UTF-8') self.assertIsNone( data['id'] ) # no ID provided, snapshot should not be saved in database self.assertTrue(data['computed']) self.assertEqual(data['computingError'], '') self.assertTrue(data['diffPixelPercentage'] > 0.000144) # check computation has been done self.assertTrue(data['tooManyDiffs']) # check temp file has been deleted self.assertFalse( os.path.isfile( os.path.join(settings.MEDIA_ROOT, 'Ibis_Mulhouse_diff.png'))) uploaded_snapshot2 = Snapshot.objects.filter( stepResult__testCase=self.tcs1, stepResult__step__id=1).last() self.assertEqual( uploaded_snapshot2, uploaded_snapshot1, "the second uploaded snapshot should not be recorded") def test_post_snapshot_no_store_picture_parameter_missing_version(self): """ Check that an error is raised when version is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test1', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_store_picture_parameter_missing_environment( self): """ Check that an error is raised when environment is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put(reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test1', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_store_picture_parameter_missing_browser(self): """ Check that an error is raised when browser is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'testCaseName': 'test1', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_store_picture_parameter_missing_test_name(self): """ Check that an error is raised when test name is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_store_picture_parameter_missing_step_name(self): """ Check that an error is raised when step name is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_store_picture_parameter_missing_image(self): """ Check that an error is raised when imag is not provided """ # check no snapshot correspond to this characteristics before the test self.assertIsNone( Snapshot.objects.filter(stepResult__testCase=self.tcs1, stepResult__step__id=1).last()) with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.put( reverse('upload', args=['img']), data={ 'name': 'img', 'compare': 'true', 'versionId': Version.objects.get(pk=1).id, 'environmentId': TestEnvironment.objects.get(pk=1).id, 'browser': 'firefox', 'testCaseName': 'test1', 'stepName': 'Step 1' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_no_picture(self): response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'name': 'img', 'compare': 'true' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_missing_step(self): with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'image': fp, 'name': 'img', 'compare': 'true' }) self.assertEqual(response.status_code, 500, 'status code should be 500') def test_post_snapshot_missing_name(self): with open('snapshotServer/tests/data/engie.png', 'rb') as fp: response = self.client.post(reverse('upload', args=['img']), data={ 'stepResult': self.sr1.id, 'image': fp, 'compare': 'true' }) self.assertEqual(response.status_code, 500, 'status code should be 500')
class TestSnapshots(SnapshotTestCase): def setUp(self): super().setUp() self.app = Application(name="test") self.app.save() self.v1 = Version(application=self.app, name='1.0') self.v1.save() self.v2 = Version(application=self.app, name='2.0') self.v2.save() env = TestEnvironment(name='DEV') env.save() self.session1 = TestSession(sessionId="1234", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v1, ttl=datetime.timedelta(0)) self.session1.save() self.session_same_env = TestSession(sessionId="1235", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v1, ttl=datetime.timedelta(0)) self.session_same_env.save() self.session3 = TestSession(sessionId="1236", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v2, ttl=datetime.timedelta(0)) self.session3.save() self.session4 = TestSession(sessionId="1237", date=datetime.datetime(2017, 5, 7, tzinfo=pytz.UTC), browser="firefox", environment=env, version=self.v2, ttl=datetime.timedelta(0)) self.session4.save() self.step = TestStep(name="step1") self.step.save() self.tc1 = TestCase(name="case1", application=self.app) self.tc1.save() self.tcs1 = TestCaseInSession(testCase=self.tc1, session=self.session1) self.tcs1.save() self.tcs_same_env = TestCaseInSession(testCase=self.tc1, session=self.session_same_env) self.tcs_same_env.save() self.tcs3 = TestCaseInSession(testCase=self.tc1, session=self.session3) self.tcs3.save() self.tcs4 = TestCaseInSession(testCase=self.tc1, session=self.session4) self.tcs4.save() self.tcs1.testSteps.set([self.step]) self.tcs1.save() self.tcs_same_env.testSteps.set([self.step]) self.tcs_same_env.save() self.tsr1 = StepResult(step=self.step, testCase=self.tcs1, result=True) self.tsr1.save() self.tsr2 = StepResult(step=self.step, testCase=self.tcs_same_env, result=True) self.tsr2.save() self.tsr3 = StepResult(step=self.step, testCase=self.tcs3, result=True) self.tsr3.save() self.tsr4 = StepResult(step=self.step, testCase=self.tcs4, result=True) self.tsr4.save() def test_no_next_snapshots(self): """ check that we do not look at ourself when searching next snapshots """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() self.assertEqual(s2.snapshotsUntilNextRef(s2.refSnapshot), [], "No next snapshot should be found") def test_too_low_diff_tolerance(self): """ tolerance < 0 should be refused """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None, diffTolerance=-0.1) self.assertRaises(IntegrityError, s1.save) def test_too_high_diff_tolerance(self): """ tolerance > 100 should be refused """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None, diffTolerance=100.1) self.assertRaises(IntegrityError, s1.save) def test_next_snapshots_with_no_ref(self): """ Search for next snapshot that reference ourself """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() # s3 should not be found has it does not reference s1 s3 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s2, pixelsDiff=None) s3.save() self.assertEqual(s1.snapshotsUntilNextRef(s1), [s2], "One snapshot should be found") def test_next_snapshots_with_ref(self): """ Check that the next reference snapshot (s4) is not rendered but pictures from the next version are """ # snapshots on app v1 s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() # snapshots on app v2 s3 = Snapshot(stepResult=self.tsr3, image=None, refSnapshot=s1, pixelsDiff=None) s3.save() s4 = Snapshot(stepResult=self.tsr4, image=None, refSnapshot=None, pixelsDiff=None) s4.save() self.assertEqual(s1.snapshotsUntilNextRef(s1), [s2, s3], "2 snapshots should be found") def test_next_snapshots_with_lower_version(self): """ We should not give snapshots from a lower version even if snapshot id is lower We assume that 2 versions are being tested at the same time. Order of declared snapshots is important """ # snapshots on app v2 s0 = Snapshot(stepResult=self.tsr3, image=None, refSnapshot=None, pixelsDiff=None) s0.save() # snapshots on app v1 s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() # snapshots on app v2 s3 = Snapshot(stepResult=self.tsr4, image=None, refSnapshot=s0, pixelsDiff=None) s3.save() self.assertEqual(s1.snapshotsUntilNextRef(s1), [s2], "One snapshot should be found") def test_snapshot_deletion_and_recomputing_on_previous_reference(self): """ Test that when snapshot is deleted and is a reference for other snapshot, this reference is removed and replaced by a previous reference when it exists Here, S1 is a reference S2 is a reference S3 has a reference on S2 After deletion of S2, S3 should have reference on S1 """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=None, pixelsDiff=None) s2.save() s3 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s2, pixelsDiff=None) s3.save() s2.delete() self.assertEqual(Snapshot.objects.get(pk=s3.id).refSnapshot, s1) def test_snapshot_deletion_of_non_reference_snapshot(self): """ Test that when snapshot is deleted and is not a reference for other snapshot, nothing happens Here, S1 is a reference S2 has a reference on S1 S3 has a reference on S1 After deletion of S2, S3 should still have reference on S1 """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() s3 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s3.save() s2.delete() self.assertEqual(Snapshot.objects.get(pk=s3.id).refSnapshot, s1) def test_snapshot_deletion_of_first_reference(self): """ Test that when first reference is deleted, other pictures have their references changed Here, S1 is a reference S2 has a reference on S1 S3 has a reference on S1 After deletion of S1, S2 becomes a reference S3 should have reference on S2 """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() s3 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s3.save() s1.delete() self.assertEqual(Snapshot.objects.get(pk=s2.id).refSnapshot, None) self.assertEqual(Snapshot.objects.get(pk=s3.id).refSnapshot, s2) def test_snapshot_deletion_of_last_snapshot(self): """ Test that nothing happens when the deleted snapshot is the last one Here, S1 is a reference S2 has a reference on S1 S3 has a reference on S1 After deletion of S1, S2 becomes a reference S3 should have reference on S2 """ s1 = Snapshot(stepResult=self.tsr1, image=None, refSnapshot=None, pixelsDiff=None) s1.save() s2 = Snapshot(stepResult=self.tsr2, image=None, refSnapshot=s1, pixelsDiff=None) s2.save() s2.delete() self.assertEqual(Snapshot.objects.get(pk=s1.id).refSnapshot, None)