def test_filename_in_error_message(self): # emulate launchable record build write_build(self.build_name) normal_xml = str( Path(__file__).parent.joinpath( '../../data/broken_xml/normal.xml').resolve()) broken_xml = str( Path(__file__).parent.joinpath( '../../data/broken_xml/broken.xml').resolve()) result = self.cli('record', 'tests', '--build', self.build_name, 'file', normal_xml, broken_xml) def remove_backslash(input: str) -> str: # Hack for Windowns. They containts double escaped backslash such as \\\\ if sys.platform == "win32": return input.replace("\\", "") else: return input # making sure the offending file path name is being printed. self.assertIn(remove_backslash(broken_xml), remove_backslash(result.output)) # normal.xml self.assertIn( 'open_class_user_test.rb', gzip.decompress(responses.calls[2].request.body).decode())
def test_subset_record_key_match(self): # emulate launchable record build write_build(self.build_name) """ Test recorded test results contain subset's test path """ result = self.cli('subset', '--target', '10%', 'bazel', input=self.subset_input) self.assertEqual(result.exit_code, 0) subset_payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) result = self.cli('record', 'tests', 'bazel', str(self.test_files_dir) + "/") self.assertEqual(result.exit_code, 0) record_payload = json.loads( gzip.decompress(responses.calls[3].request.body).decode()) record_test_paths = itertools.chain.from_iterable( e['testPath'] for e in record_payload['events']) record_test_path_dict = {t['name']: t for t in record_test_paths} for test_paths in subset_payload['testPaths']: for subset_test_path in test_paths: record_test_path = record_test_path_dict.get( subset_test_path['name']) self.assert_json_orderless_equal(record_test_path, subset_test_path)
def test_subset_split(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [[{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]], "rest": [[{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }]], "subsettingId": 123, "summary": { "subset": { "candidates": 3, "duration": 3, "rate": 75 }, "rest": { "candidate": 1, "duration": 1, "rate": 25 } }, "isBrainless": False, }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', '--split', 'gradle', str( self.test_files_dir.joinpath( 'java/app/src/test/java').resolve())) self.assertEqual(result.exit_code, 0) self.assertIn("subset/123", result.output.rstrip('\n'))
def test_subset_without_session(self): # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '10%', 'ctest', str(self.test_files_dir.joinpath("ctest_list.json"))) self.assertEqual(result.exit_code, 0) payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('subset_result.json')) self.assert_json_orderless_equal(payload, expected)
def test_record_test(self): # emulate launchable record build write_build(self.build_name) result = self.cli('record', 'tests', 'jest', str(self.test_files_dir.joinpath("junit.xml"))) self.assertEqual(result.exit_code, 0) payload = json.loads( gzip.decompress(responses.calls[2].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('record_test_result.json')) self.assert_json_orderless_equal(expected, payload)
def test_subset_without_session(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [[{ 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]], "rest": [], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 100 }, "rest": { "candidate": 0, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', 'gradle', str(self.test_files_dir.joinpath('java/app/src/test').resolve())) # TODO: we need to assert on the request payload to make sure it found test list all right self.assertEqual(result.exit_code, 0) output = '--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.sub.App3Test --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest' self.assertIn(output, result.output.rstrip('\n'))
def test_subset_without_session(self): # emulate launchable record build write_build(self.build_name) pipe = "TestExample1\nTestExample2\nTestExample3\nTestExample4\nok github.com/launchableinc/rocket-car-gotest 0.268s" result = self.cli('subset', '--target', '10%', 'go-test', input=pipe) self.assertEqual(result.exit_code, 0) self.assertEqual(read_session(self.build_name), self.session) payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('subset_result.json')) self.assert_json_orderless_equal(expected, payload)
def test_record_tests(self): with tempfile.TemporaryDirectory() as tempdir: test_path_file = os.path.join(tempdir, 'tests.json') with open(test_path_file, 'w') as f: f.write('\n'.join([ '{', ' "testCases": [', ' {', ' "testPath": "file=a.py#class=classA",', ' "duration": 42,', ' "status": "TEST_PASSED",', ' "stdout": "This is stdout",', ' "stderr": "This is stderr",', ' "createdAt": "2021-10-05T12:34:00"', ' }', ' ]', '}', ]) + '\n') # emulate launchable record build write_build(self.build_name) result = self.cli('record', 'tests', 'raw', test_path_file, mix_stderr=False) self.assertEqual(result.exit_code, 0) # Check request body payload = json.loads( gzip.decompress(responses.calls[2].request.body).decode()) self.assert_json_orderless_equal( payload, { 'events': [ { 'testPath': 'file=a.py#class=classA', 'duration': 42, 'status': 1, 'stdout': 'This is stdout', 'stderr': 'This is stderr', 'created_at': '2021-10-05T12:34:00', 'data': None, 'type': 'case', }, ], })
def test_record_test(self): # emulate launchable record build write_build(self.build_name) result = self.cli('record', 'tests', 'ctest', str(self.test_files_dir) + "/Testing/**/Test.xml") self.assertEqual(result.exit_code, 0) self.assertEqual(read_session(self.build_name), self.session) payload = json.loads( gzip.decompress(responses.calls[2].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('record_test_result.json')) for c in payload['events']: del c['created_at'] self.assert_json_orderless_equal(payload, expected)
def test_subset_split(self): test_path = Path( "{}/components/layouts/modal/snapshot.test.tsx".format( os.getcwd())) responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'name': str(test_path) }]], 'rest': [], 'subsettingId': 123, 'summary': { 'subset': { 'duration': 10, 'candidates': 1, 'rate': 100 }, 'rest': { 'duration': 0, 'candidates': 0, 'rate': 0 } }, "isBrainless": False, }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '20%', '--base', os.getcwd(), '--split', 'jest', input=self.subset_input) self.assertEqual(result.exit_code, 0) self.assertIn('subset/123', result.output)
def test_subset(self): # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '10%', '--base', os.getcwd(), 'jest', input=self.subset_input) print(result.output) self.assertEqual(result.exit_code, 0) payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('subset_result.json')) self.assert_json_orderless_equal(payload, expected)
def test_record_test_from_json(self): reports = [] for f in glob.iglob(str(self.test_files_dir.joinpath("report/*.json")), recursive=True): reports.append(f) # emulate launchable record build write_build(self.build_name) result = self.cli('record', 'tests', 'cucumber', "--json", *reports) self.assertEqual(result.exit_code, 0) payload = json.loads( gzip.decompress(responses.calls[2].request.body).decode()) for c in payload['events']: del c['created_at'] expected = self.load_json_from_file( self.test_files_dir.joinpath('record_test_json_result.json')) self.assert_json_orderless_equal(expected, payload)
def test_record_tests_without_session(self): # emulate launchable record build write_build(self.build_name) result = self.cli('record', 'tests', 'go-test', str(self.test_files_dir) + "/") self.assertEqual(result.exit_code, 0) self.assertEqual(read_session(self.build_name), self.session) self.assertIn('events', responses.calls[2].request.url, 'call events API') payload = json.loads( gzip.decompress(responses.calls[2].request.body).decode()) for c in payload['events']: del c['created_at'] expected = self.load_json_from_file( self.test_files_dir.joinpath('record_test_result.json')) self.assert_json_orderless_equal(expected, payload) self.assertIn('close', responses.calls[3].request.url, 'call close API')
def test_subset(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [ [{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], ], "rest": [ [{ 'name': 'FooTest.Baz' }], ], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 90 }, "rest": { "candidate": 1, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) with tempfile.TemporaryDirectory() as tempdir: test_path_file = os.path.join(tempdir, 'tests.txt') with open(test_path_file, 'w') as f: f.write('\n'.join([ 'testcase=FooTest.Bar', 'testcase=FooTest.Foo', '# This is a comment', 'testcase=FooTest.Baz', ]) + '\n') # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '10%', 'raw', test_path_file, mix_stderr=False) self.assertEqual(result.exit_code, 0) # Check request body payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) self.assert_json_orderless_equal( payload, { 'testPaths': [[{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], [{ 'type': 'testcase', 'name': 'FooTest.Baz' }]], 'session': { 'id': str(self.session_id) }, 'target': 0.1, }) # Check split output self.assertEqual( result.stdout, '\n'.join([ 'testcase=FooTest.Bar', 'testcase=FooTest.Foo', ]) + '\n')
def test_subset_multiple_files(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [ [{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], [{ 'type': 'testcase', 'name': '*/ParameterizedTest.Bar/*' }], ], "rest": [ [{ 'name': 'FooTest.Baz' }], ], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 90 }, "rest": { "candidate": 1, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) with tempfile.TemporaryDirectory() as tempdir: # Use a non-existing dir to check it creates a dir. output_dir = os.path.join(tempdir, 'subdir') # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', 'ctest', '--output-regex-files', '--output-regex-files-dir=' + output_dir, '--output-regex-files-size=32', str(self.test_files_dir.joinpath("ctest_list.json"))) self.assertEqual(result.exit_code, 0) subset_files = [] rest_files = [] for file in os.listdir(output_dir): with open(os.path.join(output_dir, file), 'r') as f: if file.startswith('subset'): subset_files.append(f.read().strip()) else: rest_files.append(f.read().strip()) subset_files.sort() rest_files.sort() if sys.version_info[:2] >= (3, 7): self.assertEqual(subset_files, [ '^FooTest\\.Bar$|^FooTest\\.Foo$', '^\\*/ParameterizedTest\\.Bar/\\*$' ]) else: # There was a change in re.escape behavior from Python 3.7. See # https://docs.python.org/3/library/re.html#re.escape self.assertEqual(subset_files, [ '^FooTest\\.Bar$|^FooTest\\.Foo$', '^\\*\\/ParameterizedTest\\.Bar\\/\\*$' ]) self.assertEqual(rest_files, ['^FooTest\\.Baz$'])