def test_record_session(self): build = "internal_server_error" responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions" .format(base=get_base_url(), org=self.organization, ws=self.workspace, build=build), status=500) result = self.cli("record", "session", "--build", build) self.assertEqual(result.exit_code, 0) build = "not_found" responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions" .format(base=get_base_url(), org=self.organization, ws=self.workspace, build=build), status=404) result = self.cli("record", "session", "--build", build) self.assertEqual(result.exit_code, 1)
def test_record_tests(self): test_files_dir = Path(__file__).parent.joinpath( '../data/minitest/').resolve() responses.replace( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session_id}/events" .format(base=get_base_url(), org=self.organization, ws=self.workspace, build=self.build_name, session_id=self.session_id), json=[], status=500) result = self.cli("record", "tests", "--session", self.session, "minitest", str(test_files_dir) + "/") self.assertEqual(result.exit_code, 0) responses.replace( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds/{build}/test_sessions/{session_id}/events" .format(base=get_base_url(), org=self.organization, ws=self.workspace, build=self.build_name, session_id=self.session_id), json=[], status=404) result = self.cli("record", "tests", "--session", self.session, "minitest", str(test_files_dir) + "/") self.assertEqual(result.exit_code, 0)
def setUp(self): self.maxDiff = None os.environ['LAUNCHABLE_TOKEN'] = self.launchable_token responses.add( responses.POST, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions". format(get_base_url(), self.organization, self.workspace, self.build_name), json={'id': self.session_id}, status=200) responses.add(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={'testPaths': []}, status=200) responses.add( responses.POST, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}/events" .format(get_base_url(), self.organization, self.workspace, self.build_name, self.session_id), json={}, status=200) responses.add( responses.PATCH, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}/close" .format(get_base_url(), self.organization, self.workspace, self.build_name, self.session_id), json={}, status=200)
def test_subset(self): responses.replace( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( base=get_base_url(), org=self.organization, ws=self.workspace), status=500) subset_file = "example_test.rb" with tempfile.NamedTemporaryFile(delete=False) as rest_file: result = self.cli("subset", "--target", "30%", "--session", self.session, "--rest", rest_file.name, "minitest", str(self.test_files_dir) + "/test/**/*.rb", mix_stderr=False) self.assertEqual(result.exit_code, 0) self.assertEqual(len(result.stdout.rstrip().split("\n")), 1) self.assertTrue(subset_file in result.stdout) rest = Path(rest_file.name).read_text() self.assertEqual(len(rest.rstrip().split("\n")), 1) self.assertTrue(subset_file in rest) responses.replace( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/subset".format( base=get_base_url(), org=self.organization, ws=self.workspace), status=404) with tempfile.NamedTemporaryFile(delete=False) as rest_file: result = self.cli("subset", "--target", "30%", "--session", self.session, "--rest", rest_file.name, "minitest", str(self.test_files_dir) + "/test/**/*.rb", mix_stderr=False) self.assertEqual(result.exit_code, 0) self.assertEqual(len(result.stdout.rstrip().split("\n")), 1) self.assertTrue(subset_file in result.stdout) rest = Path(rest_file.name).read_text() self.assertEqual(len(rest.rstrip().split("\n")), 1) self.assertTrue(subset_file in rest)
def test_subset_without_session(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]] }, status=200) result = self.cli( 'subset', '--target', '10%', '--build', self.build_name, 'gradle', str(self.test_files_dir.joinpath('java/app/test').resolve())) self.assertEqual(result.exit_code, 0) output = '--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.sub.App3Test --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest' self.assertEqual(result.output.rstrip('\n'), output)
def test_split_subset(self): test_path = Path("test", "example_test.rb") responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset/456/slice".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'name': str(test_path) }]], 'rest': [], 'subsettingId': 123 }, status=200) rest = tempfile.NamedTemporaryFile(delete=False) result = self.cli('split-subset', '--subset-id', 'subset/456', '--base', str(self.test_files_dir), '--bin', '2/2', '--rest', rest.name, 'minitest') self.assertEqual(result.exit_code, 0) output = Path(self.test_files_dir, "test", "example_test.rb") self.assertEqual(str(output), result.output.rstrip("\n")) self.assertEqual(rest.read().decode().rstrip("\n"), str(output)) rest.close() os.unlink(rest.name)
def test_subset_split(self): test_path = Path("test", "example_test.rb") responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'name': str(test_path) }]], 'rest': [], 'subsettingId': 123, 'summary': { 'subset': { 'duration': 10, 'candidates': 1, 'rate': 100 }, 'rest': { 'duration': 0, 'candidates': 0, 'rate': 0 } }, "isBrainless": False, }, status=200) result = self.cli('subset', '--target', '20%', '--session', self.session, '--base', str(self.test_files_dir), '--split', 'minitest', str(self.test_files_dir) + "/test/**/*.rb") self.assertEqual(result.exit_code, 0) self.assertIn('subset/123', result.output)
def test_subset(self): test_session_id = 16 responses.replace(responses.GET, "{}/intake/organizations/{}/workspaces/{}/test_sessions/{}/events".format( get_base_url(), self.organization, self.workspace, test_session_id), json=[ {"testPath": [ {"type": "file", "name": "test_file1.py"}], "duration": 1.2, "stderr": "", "stdout": "", "createdAt": "2021-01-02T03:04:05.000+00:00", "status": "SUCCESS"}, {"testPath": [ {"type": "file", "name": "test_file3.py"}], "duration": 0.6, "stderr": "", "stdout": "", "createdAt": "2021-01-02T03:04:05.000+00:00", "status": "SUCCESS"}, {"testPath": [ {"type": "file", "name": "test_file4.py"}], "duration": 1.8, "stderr": "", "stdout": "", "createdAt": "2021-01-02T03:04:05.000+00:00", "status": "FAILURE"}, {"testPath": [ {"type": "file", "name": "test_file2.py"}], "duration": 0.1, "stderr": "", "stdout": "", "createdAt": "2021-01-02T03:04:05.000+00:00", "status": "FAILURE"}, ], status=200) result = self.cli('inspect', 'tests', '--test-session-id', test_session_id, mix_stderr=False) expect = """| Test Path | Duration (sec) | Status | Uploaded At | |--------------------|------------------|----------|-------------------------------| | file=test_file1.py | 1.2 | SUCCESS | 2021-01-02T03:04:05.000+00:00 | | file=test_file3.py | 0.6 | SUCCESS | 2021-01-02T03:04:05.000+00:00 | | file=test_file4.py | 1.8 | FAILURE | 2021-01-02T03:04:05.000+00:00 | | file=test_file2.py | 0.1 | FAILURE | 2021-01-02T03:04:05.000+00:00 | """ self.assertEqual(result.stdout, expect)
def test_record_build(self): responses.add( responses.POST, "{base}/intake/organizations/{org}/workspaces/{ws}/builds".format( base=get_base_url(), org=self.organization, ws=self.workspace), status=500) result = self.cli("record", "build", "--name", "example") self.assertEqual(result.exit_code, 0)
def test_subset(self): subset_id = 456 responses.replace( responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( get_base_url(), self.organization, self.workspace, subset_id), json={ "testPaths": [ { "testPath": [{ "type": "file", "name": "test_file1.py" }], "duration": 1200 }, { "testPath": [{ "type": "file", "name": "test_file3.py" }], "duration": 600 }, ], "rest": [{ "testPath": [{ "type": "file", "name": "test_file4.py" }], "duration": 1800 }, { "testPath": [{ "type": "file", "name": "test_file2.py" }], "duration": 100 }] }, status=200) result = self.cli('inspect', 'subset', '--subset-id', subset_id, mix_stderr=False) expect = """| Order | Test Path | In Subset | Estimated duration (sec) | |---------|--------------------|-------------|----------------------------| | 1 | file=test_file1.py | ✔ | 1.2 | | 2 | file=test_file3.py | ✔ | 0.6 | | 3 | file=test_file4.py | | 1.8 | | 4 | file=test_file2.py | | 0.1 | """ self.assertEqual(result.stdout, expect)
def test_subset_split(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [[{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]], "rest": [[{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }]], "subsettingId": 123, "summary": { "subset": { "candidates": 3, "duration": 3, "rate": 75 }, "rest": { "candidate": 1, "duration": 1, "rate": 25 } }, "isBrainless": False, }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', '--split', 'gradle', str( self.test_files_dir.joinpath( 'java/app/src/test/java').resolve())) self.assertEqual(result.exit_code, 0) self.assertIn("subset/123", result.output.rstrip('\n'))
def test_subset_without_session(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [[{ 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }], [{ 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]], "rest": [], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 100 }, "rest": { "candidate": 0, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', 'gradle', str(self.test_files_dir.joinpath('java/app/src/test').resolve())) # TODO: we need to assert on the request payload to make sure it found test list all right self.assertEqual(result.exit_code, 0) output = '--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.sub.App3Test --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest' self.assertIn(output, result.output.rstrip('\n'))
def test_subset_split(self): test_path = Path( "{}/components/layouts/modal/snapshot.test.tsx".format( os.getcwd())) responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'name': str(test_path) }]], 'rest': [], 'subsettingId': 123, 'summary': { 'subset': { 'duration': 10, 'candidates': 1, 'rate': 100 }, 'rest': { 'duration': 0, 'candidates': 0, 'rate': 0 } }, "isBrainless": False, }, status=200) # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '20%', '--base', os.getcwd(), '--split', 'jest', input=self.subset_input) self.assertEqual(result.exit_code, 0) self.assertIn('subset/123', result.output)
def test_subset_rest(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]] }, status=200) rest = tempfile.NamedTemporaryFile(delete=False) result = self.cli( 'subset', '--target', '10%', '--build', self.build_name, '--rest', rest.name, 'gradle', str( self.test_files_dir.joinpath( 'java/app/src/test/java').resolve())) self.assertEqual(result.exit_code, 0) self.assertEqual( result.output.rstrip('\n'), "--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest" ) self.assertEqual( rest.read().decode(), '--tests com.launchableinc.rocket_car_gradle.sub.App3Test') rest.close() os.unlink(rest.name)
def test_split_subset(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset/456/slice".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [[{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.App2Test' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.AppTest' }], [{ 'type': 'class', 'name': 'com.launchableinc.rocket_car_gradle.utils.UtilsTest' }]], "rest": [[{ 'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test' }]] }, status=200) rest = tempfile.NamedTemporaryFile(delete=False) result = self.cli('split-subset', '--subset-id', 'subset/456', '--bin', '1/2', '--rest', rest.name, 'gradle') self.assertEqual(result.exit_code, 0) self.assertIn( "--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest", result.output.rstrip('\n')) self.assertEqual( rest.read().decode(), '--tests com.launchableinc.rocket_car_gradle.sub.App3Test') rest.close() os.unlink(rest.name)
def test_subset(self): responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format(get_base_url(), self.organization, self.workspace), json={'testPaths': [ [ {"type": "Assembly", "name": "calc.dll"}, {"type": "TestSuite", "name": "ParameterizedTests"}, {"type": "TestFixture", "name": "MyTests"}, {"type": "ParameterizedMethod", "name": "DivideTest"}, {"type": "TestCase", "name": "DivideTest(12,3)"} ], [ {"type": "Assembly", "name": "calc.dll"}, {"type": "TestSuite", "name": "calc"}, {"type": "TestFixture", "name": "Tests1"}, {"type": "TestCase", "name": "Test1"} ] ], 'rest': [], 'subsettingId': 123, 'summary': { 'subset': {'duration': 15, 'candidates': 2, 'rate': 100}, 'rest': {'duration': 0, 'candidates': 0, 'rate': 0} }, }, status=200) result = self.cli('subset', '--target', '10%', '--session', self.session, 'nunit', str(self.test_files_dir) + "/list.xml") self.assertEqual(result.exit_code, 0) payload = json.loads(gzip.decompress( responses.calls[0].request.body).decode()) expected = self.load_json_from_file( self.test_files_dir.joinpath('subset_result.json')) self.assert_json_orderless_equal(expected, payload) output = 'ParameterizedTests.MyTests.DivideTest(12,3)\ncalc.Tests1.Test1' self.assertIn(output, result.output)
def test_subset_multiple_files(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [ [{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], [{ 'type': 'testcase', 'name': '*/ParameterizedTest.Bar/*' }], ], "rest": [ [{ 'name': 'FooTest.Baz' }], ], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 90 }, "rest": { "candidate": 1, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) with tempfile.TemporaryDirectory() as tempdir: # Use a non-existing dir to check it creates a dir. output_dir = os.path.join(tempdir, 'subdir') # emulate launchable record build write_build(self.build_name) result = self.cli( 'subset', '--target', '10%', 'ctest', '--output-regex-files', '--output-regex-files-dir=' + output_dir, '--output-regex-files-size=32', str(self.test_files_dir.joinpath("ctest_list.json"))) self.assertEqual(result.exit_code, 0) subset_files = [] rest_files = [] for file in os.listdir(output_dir): with open(os.path.join(output_dir, file), 'r') as f: if file.startswith('subset'): subset_files.append(f.read().strip()) else: rest_files.append(f.read().strip()) subset_files.sort() rest_files.sort() if sys.version_info[:2] >= (3, 7): self.assertEqual(subset_files, [ '^FooTest\\.Bar$|^FooTest\\.Foo$', '^\\*/ParameterizedTest\\.Bar/\\*$' ]) else: # There was a change in re.escape behavior from Python 3.7. See # https://docs.python.org/3/library/re.html#re.escape self.assertEqual(subset_files, [ '^FooTest\\.Bar$|^FooTest\\.Foo$', '^\\*\\/ParameterizedTest\\.Bar\\/\\*$' ]) self.assertEqual(rest_files, ['^FooTest\\.Baz$'])
def setUp(self): self.dir = tempfile.mkdtemp() os.environ[SESSION_DIR_KEY] = self.dir self.maxDiff = None responses.add( responses.POST, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions". format(get_base_url(), self.organization, self.workspace, self.build_name), json={'id': self.session_id}, status=200) responses.add(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ 'testPaths': [], 'rest': [], 'subsettingId': 456 }, status=200) responses.add( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset/{}/slice".format( get_base_url(), self.organization, self.workspace, self.subsetting_id), json={ 'testPaths': [], 'rest': [], 'subsettingId': 456 }, status=200) responses.add( responses.GET, "{}/intake/organizations/{}/workspaces/{}/subset/{}".format( get_base_url(), self.organization, self.workspace, self.subsetting_id), json={ 'testPaths': [], 'rest': [], 'subsettingId': 456 }, status=200) responses.add( responses.POST, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}/events" .format(get_base_url(), self.organization, self.workspace, self.build_name, self.session_id), json={}, status=200) responses.add( responses.GET, "{}/intake/organizations/{}/workspaces/{}/test_sessions/{}/events". format(get_base_url(), self.organization, self.workspace, self.session_id), json=[], status=200) responses.add( responses.PATCH, "{}/intake/organizations/{}/workspaces/{}/builds/{}/test_sessions/{}/close" .format(get_base_url(), self.organization, self.workspace, self.build_name, self.session_id), json={}, status=200) responses.add( responses.GET, "{}/intake/organizations/{}/workspaces/{}/builds/{}".format( get_base_url(), self.organization, self.workspace, self.build_name), json={ 'createdAt': "2020-01-02T03:45:56.123+00:00", 'id': 123 }, status=200)
def test_subset(self): responses.replace( responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset".format( get_base_url(), self.organization, self.workspace), json={ "testPaths": [ [{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], ], "rest": [ [{ 'name': 'FooTest.Baz' }], ], "subsettingId": 456, "summary": { "subset": { "candidates": 4, "duration": 4, "rate": 90 }, "rest": { "candidate": 1, "duration": 0, "rate": 0 } }, "isBrainless": False }, status=200) with tempfile.TemporaryDirectory() as tempdir: test_path_file = os.path.join(tempdir, 'tests.txt') with open(test_path_file, 'w') as f: f.write('\n'.join([ 'testcase=FooTest.Bar', 'testcase=FooTest.Foo', '# This is a comment', 'testcase=FooTest.Baz', ]) + '\n') # emulate launchable record build write_build(self.build_name) result = self.cli('subset', '--target', '10%', 'raw', test_path_file, mix_stderr=False) self.assertEqual(result.exit_code, 0) # Check request body payload = json.loads( gzip.decompress(responses.calls[1].request.body).decode()) self.assert_json_orderless_equal( payload, { 'testPaths': [[{ 'type': 'testcase', 'name': 'FooTest.Bar' }], [{ 'type': 'testcase', 'name': 'FooTest.Foo' }], [{ 'type': 'testcase', 'name': 'FooTest.Baz' }]], 'session': { 'id': str(self.session_id) }, 'target': 0.1, }) # Check split output self.assertEqual( result.stdout, '\n'.join([ 'testcase=FooTest.Bar', 'testcase=FooTest.Foo', ]) + '\n')
def test_split_subset(self): responses.replace(responses.POST, "{}/intake/organizations/{}/workspaces/{}/subset/456/slice".format(get_base_url(), self.organization, self.workspace), json={ 'testPaths': [ [ {"type": "Assembly", "name": "calc.dll"}, {"type": "TestSuite", "name": "ParameterizedTests"}, {"type": "TestFixture", "name": "MyTests"}, {"type": "ParameterizedMethod", "name": "DivideTest"}, {"type": "TestCase", "name": "DivideTest(12,3)"} ] ], 'rest': [[ {"type": "Assembly", "name": "calc.dll"}, {"type": "TestSuite", "name": "calc"}, {"type": "TestFixture", "name": "Tests1"}, {"type": "TestCase", "name": "Test1"} ]], 'subsettingId': 456, 'summary': { 'subset': {'duration': 8, 'candidates': 1, 'rate': 50}, 'rest': {'duration': 7, 'candidates': 1, 'rate': 50} }, }, status=200) rest = tempfile.NamedTemporaryFile(delete=False) result = self.cli('split-subset', '--subset-id', 'subset/456', '--bin', '1/2', '--rest', rest.name, 'nunit') self.assertEqual(result.exit_code, 0) self.assertIn('ParameterizedTests.MyTests.DivideTest(12,3)', result.output) self.assertEqual(rest.read().decode(), 'calc.Tests1.Test1') rest.close() os.unlink(rest.name)