class NormalPhasePlugin:
"""Plugin class for pytest"""
def __init__(self, test_set):
"""Initialize database connection"""
self.test_func_times = {}
self.test_set = test_set
self.database = DatabaseHelper()
self.database.init_conn()
self.fill_times_dict()
def fill_times_dict(self):
"""Query running times of tests from database"""
for testname in self.test_set:
self.test_func_times[testname] = self.database.get_test_duration(
testname)
def pytest_collection_modifyitems(self, session, config, items):
"""Select only specific tests for running and prioritize them based on queried times"""
del config
original_length = len(items)
selected = []
for item in items:
if item.nodeid in self.test_set:
selected.append(item)
items[:] = sorted(selected,
key=lambda item: self.test_func_times[item.nodeid])
session.config.hook.pytest_deselected(
items=([FakeItem(session.config)] *
(original_length - len(selected))))
def test_newly_added_tests(helper):
db = DatabaseHelper()
db.init_conn()
helper.change_file("changes/test_car/add_test_passengers.txt", "tests/test_car.py")
new_tests = common.read_newly_added_tests(db)
db.close_conn()
assert new_tests == {"tests/test_car.py::test_passengers"}
class InitPhasePlugin:
"""Class to handle mapping database initialization"""
def __init__(self):
""""Constructor calls database and Coverage.py initialization"""
self.test_func_lines = {}
self.cov = coverage.Coverage()
self.cov._warn_unimported_source = False
self.testfiles = set()
self.database = DatabaseHelper()
self.database.init_conn()
self.database.init_mapping_db()
self.head_hash = get_current_head_hash()
self.database.save_last_update_hash(self.head_hash)
def pytest_collection_modifyitems(self, session, config, items):
"""Calculate function start and end line numbers from testfiles"""
del session, config
for item in items:
testfile = item.nodeid.split("::")[0]
self.testfiles.add(testfile)
if testfile not in self.test_func_lines:
testfile_src_code = coverage.python.get_python_source(testfile)
self.test_func_lines[testfile] = calculate_func_lines(
testfile_src_code)
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_protocol(self, item, nextitem):
"""Start coverage collection for each test function run and save data"""
del nextitem
if isinstance(item, Function):
start = timer()
self.cov.erase()
self.cov.start()
yield
self.cov.stop()
self.cov.save()
end = timer()
elapsed = round(end - start, 4)
_, test_function_id = save_testfile_and_func_data(
item, elapsed, self.test_func_lines, self.database)
save_mapping_data(
test_function_id,
self.cov.get_data(),
self.testfiles,
self.database,
)
else:
yield
def test_split_changes():
db = DatabaseHelper()
db.init_conn()
change_list = ["src/car.py", "random_file.txt", "tests/test_car.py"]
for path in change_list:
with open(path, "w") as f:
f.write("nothing")
test_files, src_files = common.split_changes(change_list, db)
db.close_conn()
for t in test_files:
assert t[1] in {"tests/test_car.py"}
for s in src_files:
assert s[1] in {"src/car.py"}
def test_tests_from_changed_sourcefiles(helper):
db = DatabaseHelper()
db.init_conn()
helper.change_file("changes/car/change_accelerate.txt", "src/car.py")
changed_files = ["src/car.py"]
test_files, src_files = common.split_changes(changed_files, db)
diff_dict = common.file_diff_dict_current(src_files)
(
test_set,
changed_lines_dict,
new_line_map_dict,
files_to_warn,
) = common.tests_from_changed_srcfiles(diff_dict, src_files, db)
db.close_conn()
assert test_set == {"tests/test_car.py::test_acceleration"}
def test_delete_ran_lines():
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
file_id = 1
sql = "SELECT line_id FROM test_map WHERE file_id = ?"
old_line_ids = [x[0] for x in c.execute(sql, (file_id, )).fetchall()]
conn.close()
db = DatabaseHelper()
db.init_conn()
db.delete_ran_lines(old_line_ids, file_id)
db.close_conn()
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
new_line_ids = [x[0] for x in c.execute(sql, (file_id, )).fetchall()]
conn.close()
assert not new_line_ids
def main():
"""Collect newly added tests and store them to database"""
db_helper = DatabaseHelper()
db_helper.init_conn()
existing_tests = set()
for test in [
x[0]
for x in db_helper.db_cursor.execute(
"SELECT context FROM test_function"
).fetchall()
]:
existing_tests.add(test)
test_set = newly_added_tests(existing_tests)
db_helper.db_cursor.execute("DELETE FROM new_tests")
for test in test_set:
db_helper.db_cursor.execute(
"INSERT INTO new_tests (context) VALUES (?)", (test,)
)
db_helper.db_conn.commit()
db_helper.db_conn.close()
def test_update_db_from_test_mapping():
# shift all lines and compare old and new lines in db
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
testfile_id = 2
funclines_sql = "SELECT id,start,end FROM test_function WHERE test_file_id = ?"
filename_sql = "SELECT path FROM test_file WHERE id = ?"
testfile_name = c.execute(filename_sql, (testfile_id, )).fetchone()[0]
old_func_line_dict = {
x[0]: (x[1], x[2])
for x in c.execute(funclines_sql, (testfile_id, )).fetchall()
}
conn.close()
shift = 5
line_count = sum(1 for line in open(testfile_name))
line_map = {x: x + shift for x in range(1, line_count + 1)}
db = DatabaseHelper()
db.init_conn()
db.update_db_from_test_mapping(line_map, testfile_id)
db.close_conn()
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
new_func_line_dict = {
x[0]: (x[1], x[2])
for x in c.execute(funclines_sql, (testfile_id, )).fetchall()
}
conn.close()
assert len(old_func_line_dict) == len(new_func_line_dict)
for key in old_func_line_dict.keys():
old_func_linenumbers = old_func_line_dict[key]
new_func_linenumbers = new_func_line_dict[key]
assert old_func_linenumbers[0] + shift == new_func_linenumbers[0]
assert old_func_linenumbers[1] + shift == new_func_linenumbers[1]
def test_tests_from_changed_testfiles(helper):
db = DatabaseHelper()
db.init_conn()
helper.change_file(
"changes/test_shop/change_test_normal_shop_purchase.txt",
"tests/test_shop.py",
)
changed_files = [
"tests/test_shop.py",
]
test_files, src_files = common.split_changes(changed_files, db)
diff_dict = common.file_diff_dict_current(test_files)
(
test_set,
changed_lines_dict,
new_line_map_dict,
) = common.tests_from_changed_testfiles(diff_dict, test_files, db)
db.close_conn()
assert test_set == {"tests/test_shop.py::test_normal_shop_purchase"}
def test_update_db_from_src_mapping():
# shift all lines and compare old and new lines in db
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
file_id = 1
sql = "SELECT line_id FROM test_map WHERE file_id = ?"
old_line_ids = [x[0] for x in c.execute(sql, (file_id, )).fetchall()]
conn.close()
shift = 5
line_map = {x: x + shift for x in old_line_ids}
db = DatabaseHelper()
db.init_conn()
db.update_db_from_src_mapping(line_map, file_id)
db.close_conn()
conn = sqlite3.connect(DB_FILE_NAME)
c = conn.cursor()
new_line_ids = [x[0] for x in c.execute(sql, (file_id, )).fetchall()]
conn.close()
assert new_line_ids == [k + shift for k in old_line_ids]
def get_newly_added_tests_from_tool(self):
db = DatabaseHelper()
db.init_conn()
new_tests = read_newly_added_tests(db)
db.close_conn()
return new_tests
def get_all_tests_for_srcfile(self, src_file_id):
db = DatabaseHelper()
db.init_conn()
all_tests = db.query_all_tests_srcfile(src_file_id)
db.close_conn()
return all_tests
def get_tests_from_tool_committed(self):
db = DatabaseHelper()
db.init_conn()
change_data = get_tests_and_data_committed(db)
db.close_conn()
return change_data.test_set
def random_remove_test(iterations, deletes_per_iteration, max_wait, logger):
"""Delete random lines and evaluate tests sets and pytest exitcodes"""
if not os.path.isfile(DB_FILE_NAME):
logger.info("Running mapping database initialization...")
subprocess.run(["pytest", "--rts"], check=False)
results_db = ResultDatabase()
results_db.init_conn()
results_db.init_results_db()
mapping_db = DatabaseHelper()
mapping_db.init_conn()
test_suite_size = mapping_db.get_test_suite_size()
project_name = os.getcwd()
init_hash = mapping_db.get_last_update_hash()
db_size = os.path.getsize("./mapping.db")
project_id = results_db.store_results_project(project_name, init_hash,
test_suite_size, db_size)
_, src_files = mapping_db.get_testfiles_and_srcfiles()
testhelper = TestHelper()
for i in range(iterations):
# Remove random lines
testhelper.checkout_new_branch()
for j in range(deletes_per_iteration):
random_file = select_random_file(src_files)
filename = random_file[1]
delete_random_line(filename)
testhelper.commit_change(filename, str(j + 1))
# Gets tests based on line-level and file-level change
current_git_hash = get_current_head_hash()
changed_files = changed_files_between_commits(init_hash,
current_git_hash)
tests_line_level = set()
tests_file_level = set()
for filename in changed_files:
diff = file_diff_data_between_commits(filename, init_hash,
current_git_hash)
test_lines, _, _ = get_test_lines_and_update_lines(diff)
file_id = mapping_db.save_src_file(filename)
tests_line = mapping_db.query_tests_srcfile(test_lines, file_id)
tests_file = mapping_db.query_all_tests_srcfile(file_id)
for testfunc_line in tests_line:
tests_line_level.add(testfunc_line)
for testfunc_file in tests_file:
tests_file_level.add(testfunc_file)
# Get full git diff for analysis
full_diff = full_diff_between_commits(init_hash, current_git_hash)
# Pytest exitcodes for running different test sets
exitcode_line = (capture_specific_exit_code(
list(tests_line_level), max_wait) if tests_line_level else 5)
exitcode_file = (capture_specific_exit_code(
list(tests_file_level), max_wait) if tests_file_level else 5)
exitcode_all = capture_all_exit_code(max_wait)
# Clear removal
testhelper.checkout_branch("master")
testhelper.delete_branch("new-branch")
# Store and print data
results_db.store_results_data(
project_id,
deletes_per_iteration,
exitcode_line,
exitcode_file,
exitcode_all,
len(tests_line_level),
len(tests_file_level),
full_diff,
)
print_remove_test_output(
i,
project_name,
init_hash,
deletes_per_iteration,
test_suite_size,
len(tests_line_level),
len(tests_file_level),
exitcode_line,
exitcode_file,
exitcode_all,
RESULTS_DB_FILE_NAME,
logger,
)
def pytest_configure(config):
"""Register RTS plugins based on state"""
logger = logging.getLogger()
logging.basicConfig(format="%(message)s", level=logging.INFO)
if config.option.rts:
if not os.path.isfile(DB_FILE_NAME):
logger.info("No mapping database detected, starting initialization...")
config.pluginmanager.register(InitPhasePlugin())
return
db_helper = DatabaseHelper()
db_helper.init_conn()
workdir_data = get_tests_and_data_current(db_helper)
logger.info("WORKING DIRECTORY CHANGES")
logger.info(
"Found %s changed test files", workdir_data.changed_testfiles_amount
)
logger.info("Found %s changed src files", workdir_data.changed_srcfiles_amount)
logger.info("Found %s tests to execute\n", len(workdir_data.test_set))
if workdir_data.test_set:
logger.info(
"Running WORKING DIRECTORY test set and exiting without updating..."
)
config.pluginmanager.register(NormalPhasePlugin(workdir_data.test_set))
return
logger.info("No WORKING DIRECTORY tests to run, checking COMMITTED changes...")
current_hash = get_current_head_hash()
if db_helper.is_last_update_hash(current_hash):
pytest.exit("Database is updated to the current commit state", 0)
previous_hash = db_helper.get_last_update_hash()
logger.info("Comparison: %s\n", " => ".join([current_hash, previous_hash]))
committed_data = get_tests_and_data_committed(db_helper)
logger.info("COMMITTED CHANGES")
logger.info(
"Found %s changed test files", committed_data.changed_testfiles_amount
)
logger.info(
"Found %s changed src files", committed_data.changed_srcfiles_amount
)
logger.info(
"Found %s newly added tests",
committed_data.new_tests_amount,
)
logger.info("Found %s tests to execute\n", len(committed_data.test_set))
if committed_data.warning_needed:
logger.info(
"WARNING: New lines were added to the following files but no new tests discovered:"
)
logger.info("\n".join(committed_data.files_to_warn))
logger.info("=> Executing tests (if any) and updating database")
db_helper.save_last_update_hash(current_hash)
update_mapping_db(committed_data.update_data, db_helper)
if committed_data.test_set:
config.pluginmanager.register(UpdatePhasePlugin(committed_data.test_set))
return
pytest.exit("No tests to run", 0)