def test_filename_in_error_message(self):
# emulate launchable record build
write_build(self.build_name)
normal_xml = str(
Path(__file__).parent.joinpath(
'../../data/broken_xml/normal.xml').resolve())
broken_xml = str(
Path(__file__).parent.joinpath(
'../../data/broken_xml/broken.xml').resolve())
result = self.cli('record', 'tests', '--build', self.build_name,
'file', normal_xml, broken_xml)
def remove_backslash(input: str) -> str:
# Hack for Windowns. They containts double escaped backslash such as \\\\
if sys.platform == "win32":
return input.replace("\\", "")
else:
return input
# making sure the offending file path name is being printed.
self.assertIn(remove_backslash(broken_xml),
remove_backslash(result.output))
# normal.xml
self.assertIn(
'open_class_user_test.rb',
gzip.decompress(responses.calls[2].request.body).decode())
def test_subset_record_key_match(self):
# emulate launchable record build
write_build(self.build_name)
"""
Test recorded test results contain subset's test path
"""
result = self.cli('subset',
'--target',
'10%',
'bazel',
input=self.subset_input)
self.assertEqual(result.exit_code, 0)
subset_payload = json.loads(
gzip.decompress(responses.calls[1].request.body).decode())
result = self.cli('record', 'tests', 'bazel',
str(self.test_files_dir) + "/")
self.assertEqual(result.exit_code, 0)
record_payload = json.loads(
gzip.decompress(responses.calls[3].request.body).decode())
record_test_paths = itertools.chain.from_iterable(
e['testPath'] for e in record_payload['events'])
record_test_path_dict = {t['name']: t for t in record_test_paths}
for test_paths in subset_payload['testPaths']:
for subset_test_path in test_paths:
record_test_path = record_test_path_dict.get(
subset_test_path['name'])
self.assert_json_orderless_equal(record_test_path,
subset_test_path)
def test_subset_split(self):
responses.replace(
responses.POST,
"{}/intake/organizations/{}/workspaces/{}/subset".format(
get_base_url(), self.organization, self.workspace),
json={
"testPaths":
[[{
'type': 'class',
'name': 'com.launchableinc.rocket_car_gradle.App2Test'
}],
[{
'type': 'class',
'name': 'com.launchableinc.rocket_car_gradle.AppTest'
}],
[{
'type': 'class',
'name':
'com.launchableinc.rocket_car_gradle.utils.UtilsTest'
}]],
"rest": [[{
'name':
'com.launchableinc.rocket_car_gradle.sub.App3Test'
}]],
"subsettingId":
123,
"summary": {
"subset": {
"candidates": 3,
"duration": 3,
"rate": 75
},
"rest": {
"candidate": 1,
"duration": 1,
"rate": 25
}
},
"isBrainless":
False,
},
status=200)
# emulate launchable record build
write_build(self.build_name)
result = self.cli(
'subset', '--target', '10%', '--split', 'gradle',
str(
self.test_files_dir.joinpath(
'java/app/src/test/java').resolve()))
self.assertEqual(result.exit_code, 0)
self.assertIn("subset/123", result.output.rstrip('\n'))
def test_subset_without_session(self):
# emulate launchable record build
write_build(self.build_name)
result = self.cli('subset', '--target', '10%', 'ctest',
str(self.test_files_dir.joinpath("ctest_list.json")))
self.assertEqual(result.exit_code, 0)
payload = json.loads(
gzip.decompress(responses.calls[1].request.body).decode())
expected = self.load_json_from_file(
self.test_files_dir.joinpath('subset_result.json'))
self.assert_json_orderless_equal(payload, expected)
def test_record_test(self):
# emulate launchable record build
write_build(self.build_name)
result = self.cli('record', 'tests', 'jest',
str(self.test_files_dir.joinpath("junit.xml")))
self.assertEqual(result.exit_code, 0)
payload = json.loads(
gzip.decompress(responses.calls[2].request.body).decode())
expected = self.load_json_from_file(
self.test_files_dir.joinpath('record_test_result.json'))
self.assert_json_orderless_equal(expected, payload)
def test_subset_without_session(self):
responses.replace(
responses.POST,
"{}/intake/organizations/{}/workspaces/{}/subset".format(
get_base_url(), self.organization, self.workspace),
json={
"testPaths":
[[{
'name': 'com.launchableinc.rocket_car_gradle.App2Test'
}], [{
'name': 'com.launchableinc.rocket_car_gradle.AppTest'
}],
[{
'name': 'com.launchableinc.rocket_car_gradle.sub.App3Test'
}],
[{
'name':
'com.launchableinc.rocket_car_gradle.utils.UtilsTest'
}]],
"rest": [],
"subsettingId":
456,
"summary": {
"subset": {
"candidates": 4,
"duration": 4,
"rate": 100
},
"rest": {
"candidate": 0,
"duration": 0,
"rate": 0
}
},
"isBrainless":
False
},
status=200)
# emulate launchable record build
write_build(self.build_name)
result = self.cli(
'subset', '--target', '10%', 'gradle',
str(self.test_files_dir.joinpath('java/app/src/test').resolve()))
# TODO: we need to assert on the request payload to make sure it found test list all right
self.assertEqual(result.exit_code, 0)
output = '--tests com.launchableinc.rocket_car_gradle.App2Test --tests com.launchableinc.rocket_car_gradle.AppTest --tests com.launchableinc.rocket_car_gradle.sub.App3Test --tests com.launchableinc.rocket_car_gradle.utils.UtilsTest'
self.assertIn(output, result.output.rstrip('\n'))
def test_subset_without_session(self):
# emulate launchable record build
write_build(self.build_name)
pipe = "TestExample1\nTestExample2\nTestExample3\nTestExample4\nok github.com/launchableinc/rocket-car-gotest 0.268s"
result = self.cli('subset', '--target', '10%', 'go-test', input=pipe)
self.assertEqual(result.exit_code, 0)
self.assertEqual(read_session(self.build_name), self.session)
payload = json.loads(
gzip.decompress(responses.calls[1].request.body).decode())
expected = self.load_json_from_file(
self.test_files_dir.joinpath('subset_result.json'))
self.assert_json_orderless_equal(expected, payload)
def test_record_tests(self):
with tempfile.TemporaryDirectory() as tempdir:
test_path_file = os.path.join(tempdir, 'tests.json')
with open(test_path_file, 'w') as f:
f.write('\n'.join([
'{',
' "testCases": [',
' {',
' "testPath": "file=a.py#class=classA",',
' "duration": 42,',
' "status": "TEST_PASSED",',
' "stdout": "This is stdout",',
' "stderr": "This is stderr",',
' "createdAt": "2021-10-05T12:34:00"',
' }',
' ]',
'}',
]) + '\n')
# emulate launchable record build
write_build(self.build_name)
result = self.cli('record',
'tests',
'raw',
test_path_file,
mix_stderr=False)
self.assertEqual(result.exit_code, 0)
# Check request body
payload = json.loads(
gzip.decompress(responses.calls[2].request.body).decode())
self.assert_json_orderless_equal(
payload, {
'events': [
{
'testPath': 'file=a.py#class=classA',
'duration': 42,
'status': 1,
'stdout': 'This is stdout',
'stderr': 'This is stderr',
'created_at': '2021-10-05T12:34:00',
'data': None,
'type': 'case',
},
],
})
def test_record_test(self):
# emulate launchable record build
write_build(self.build_name)
result = self.cli('record', 'tests', 'ctest',
str(self.test_files_dir) + "/Testing/**/Test.xml")
self.assertEqual(result.exit_code, 0)
self.assertEqual(read_session(self.build_name), self.session)
payload = json.loads(
gzip.decompress(responses.calls[2].request.body).decode())
expected = self.load_json_from_file(
self.test_files_dir.joinpath('record_test_result.json'))
for c in payload['events']:
del c['created_at']
self.assert_json_orderless_equal(payload, expected)
def test_subset_split(self):
test_path = Path(
"{}/components/layouts/modal/snapshot.test.tsx".format(
os.getcwd()))
responses.replace(
responses.POST,
"{}/intake/organizations/{}/workspaces/{}/subset".format(
get_base_url(), self.organization, self.workspace),
json={
'testPaths': [[{
'name': str(test_path)
}]],
'rest': [],
'subsettingId': 123,
'summary': {
'subset': {
'duration': 10,
'candidates': 1,
'rate': 100
},
'rest': {
'duration': 0,
'candidates': 0,
'rate': 0
}
},
"isBrainless": False,
},
status=200)
# emulate launchable record build
write_build(self.build_name)
result = self.cli('subset',
'--target',
'20%',
'--base',
os.getcwd(),
'--split',
'jest',
input=self.subset_input)
self.assertEqual(result.exit_code, 0)
self.assertIn('subset/123', result.output)
def test_subset(self):
# emulate launchable record build
write_build(self.build_name)
result = self.cli('subset',
'--target',
'10%',
'--base',
os.getcwd(),
'jest',
input=self.subset_input)
print(result.output)
self.assertEqual(result.exit_code, 0)
payload = json.loads(
gzip.decompress(responses.calls[1].request.body).decode())
expected = self.load_json_from_file(
self.test_files_dir.joinpath('subset_result.json'))
self.assert_json_orderless_equal(payload, expected)
def test_record_test_from_json(self):
reports = []
for f in glob.iglob(str(self.test_files_dir.joinpath("report/*.json")),
recursive=True):
reports.append(f)
# emulate launchable record build
write_build(self.build_name)
result = self.cli('record', 'tests', 'cucumber', "--json", *reports)
self.assertEqual(result.exit_code, 0)
payload = json.loads(
gzip.decompress(responses.calls[2].request.body).decode())
for c in payload['events']:
del c['created_at']
expected = self.load_json_from_file(
self.test_files_dir.joinpath('record_test_json_result.json'))
self.assert_json_orderless_equal(expected, payload)
def test_record_tests_without_session(self):
# emulate launchable record build
write_build(self.build_name)
result = self.cli('record', 'tests', 'go-test',
str(self.test_files_dir) + "/")
self.assertEqual(result.exit_code, 0)
self.assertEqual(read_session(self.build_name), self.session)
self.assertIn('events', responses.calls[2].request.url,
'call events API')
payload = json.loads(
gzip.decompress(responses.calls[2].request.body).decode())
for c in payload['events']:
del c['created_at']
expected = self.load_json_from_file(
self.test_files_dir.joinpath('record_test_result.json'))
self.assert_json_orderless_equal(expected, payload)
self.assertIn('close', responses.calls[3].request.url,
'call close API')
def test_subset(self):
responses.replace(
responses.POST,
"{}/intake/organizations/{}/workspaces/{}/subset".format(
get_base_url(), self.organization, self.workspace),
json={
"testPaths": [
[{
'type': 'testcase',
'name': 'FooTest.Bar'
}],
[{
'type': 'testcase',
'name': 'FooTest.Foo'
}],
],
"rest": [
[{
'name': 'FooTest.Baz'
}],
],
"subsettingId":
456,
"summary": {
"subset": {
"candidates": 4,
"duration": 4,
"rate": 90
},
"rest": {
"candidate": 1,
"duration": 0,
"rate": 0
}
},
"isBrainless":
False
},
status=200)
with tempfile.TemporaryDirectory() as tempdir:
test_path_file = os.path.join(tempdir, 'tests.txt')
with open(test_path_file, 'w') as f:
f.write('\n'.join([
'testcase=FooTest.Bar',
'testcase=FooTest.Foo',
'# This is a comment',
'testcase=FooTest.Baz',
]) + '\n')
# emulate launchable record build
write_build(self.build_name)
result = self.cli('subset',
'--target',
'10%',
'raw',
test_path_file,
mix_stderr=False)
self.assertEqual(result.exit_code, 0)
# Check request body
payload = json.loads(
gzip.decompress(responses.calls[1].request.body).decode())
self.assert_json_orderless_equal(
payload, {
'testPaths': [[{
'type': 'testcase',
'name': 'FooTest.Bar'
}], [{
'type': 'testcase',
'name': 'FooTest.Foo'
}], [{
'type': 'testcase',
'name': 'FooTest.Baz'
}]],
'session': {
'id': str(self.session_id)
},
'target':
0.1,
})
# Check split output
self.assertEqual(
result.stdout, '\n'.join([
'testcase=FooTest.Bar',
'testcase=FooTest.Foo',
]) + '\n')
def test_subset_multiple_files(self):
responses.replace(
responses.POST,
"{}/intake/organizations/{}/workspaces/{}/subset".format(
get_base_url(), self.organization, self.workspace),
json={
"testPaths": [
[{
'type': 'testcase',
'name': 'FooTest.Bar'
}],
[{
'type': 'testcase',
'name': 'FooTest.Foo'
}],
[{
'type': 'testcase',
'name': '*/ParameterizedTest.Bar/*'
}],
],
"rest": [
[{
'name': 'FooTest.Baz'
}],
],
"subsettingId":
456,
"summary": {
"subset": {
"candidates": 4,
"duration": 4,
"rate": 90
},
"rest": {
"candidate": 1,
"duration": 0,
"rate": 0
}
},
"isBrainless":
False
},
status=200)
with tempfile.TemporaryDirectory() as tempdir:
# Use a non-existing dir to check it creates a dir.
output_dir = os.path.join(tempdir, 'subdir')
# emulate launchable record build
write_build(self.build_name)
result = self.cli(
'subset', '--target', '10%', 'ctest', '--output-regex-files',
'--output-regex-files-dir=' + output_dir,
'--output-regex-files-size=32',
str(self.test_files_dir.joinpath("ctest_list.json")))
self.assertEqual(result.exit_code, 0)
subset_files = []
rest_files = []
for file in os.listdir(output_dir):
with open(os.path.join(output_dir, file), 'r') as f:
if file.startswith('subset'):
subset_files.append(f.read().strip())
else:
rest_files.append(f.read().strip())
subset_files.sort()
rest_files.sort()
if sys.version_info[:2] >= (3, 7):
self.assertEqual(subset_files, [
'^FooTest\\.Bar$|^FooTest\\.Foo$',
'^\\*/ParameterizedTest\\.Bar/\\*$'
])
else:
# There was a change in re.escape behavior from Python 3.7. See
# https://docs.python.org/3/library/re.html#re.escape
self.assertEqual(subset_files, [
'^FooTest\\.Bar$|^FooTest\\.Foo$',
'^\\*\\/ParameterizedTest\\.Bar\\/\\*$'
])
self.assertEqual(rest_files, ['^FooTest\\.Baz$'])