def test_remove_old(monkeypatch): monkeypatch.setattr(bpo.config.const.images, "images", { "qemu-amd64": { "branches": ["master"], "branch_configs": { "master": {"ui": ["sxmo"], "keep": 2} } }}) branch = "master" device = "qemu-amd64" ui = "sxmo" # Init and clear DB monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.BPOServer() session = bpo.db.session() # Add four test images (similar to job_callback_build_image()) for i in range(1, 5): date = datetime.datetime.fromisoformat(f"2021-01-0{i}") dir_name = f"2021010{i}-1337" # Add to DB image = bpo.db.Image(device, branch, ui) bpo.db.set_image_status(session, image, bpo.db.ImageStatus.published, job_id=i, dir_name=dir_name, date=date) # Create dir with index.html path_img = bpo.images.path_db_obj(image) os.makedirs(path_img) bpo.ui.images.write_index_file_list(path_img, "05_files.html") # Verify that image dirs were created dir_ui = f"{bpo.config.args.images_path}/edge/qemu-amd64/sxmo" assert os.path.exists(f"{dir_ui}/20210101-1337/index.html") assert os.path.exists(f"{dir_ui}/20210102-1337/index.html") assert os.path.exists(f"{dir_ui}/20210103-1337/index.html") assert os.path.exists(f"{dir_ui}/20210104-1337/index.html") # Run remove_old with keep=2 bpo.images.remove_old() # Verify that the two oldest image dir were removed assert not os.path.exists(f"{dir_ui}/20210101-1337") assert not os.path.exists(f"{dir_ui}/20210102-1337") assert os.path.exists(f"{dir_ui}/20210103-1337/index.html") assert os.path.exists(f"{dir_ui}/20210104-1337/index.html")
def test_depends_SLOW_60s(monkeypatch): """ Trigger the api push hook, then let bpo run the depends job. Monkeypatch bpo.repo.build, so it stops after receiving depends and does not try to build the repo. """ # Limit to two arches (more would increase test time) branches = collections.OrderedDict() branches["master"] = { "arches": ["x86_64", "armv7"], "ignore_errors": False } monkeypatch.setattr(bpo.config.const, "branches", branches) with bpo_test.BPOServer(): monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.trigger.push_hook_gitlab()
def test_build_thread_safety(monkeypatch): """ Verify that bpo.repo.build doesn't run more than once at a time, even if both MainThread and ImageTimerThread happen to call it at the same time. Expected timeline: ImageTimerThread |..|bb|bb|bb|..|ww|ww|bb|bb|bb|..|ww|ww| MainThread |..|..|ww|ww|bb|bb|bb|..|..|ww|bb|bb|bb| |--|--|--|--|--|--|--|--|--|--|--|--|--|>time in 0.01s 00 01 02 03 04 05 06 07 08 09 10 11 12 13 |..|: time.sleep in MainThread / timer interval sleep in ImageTimerThread |bb|: running bpo.repo.build (build_dummy below) |ww|: waiting for bpo.repo.build due to lock If the lock did not work, the ImageTimerThread would run not just two, but three times (take out the |ww| blocks). Therefore the test shows that the function is thread safe. """ bpo_test.BPOServer() # The order in which the threads run bpo.repo.build threads_expected = [ "ImageTimerThread", "MainThread", "ImageTimerThread", "MainThread" ] threads = [] def build_dummy(force_repo_update=False, no_repo_update=False): logging.info("build_dummy called") threads.append(threading.current_thread().name) time.sleep(0.03) monkeypatch.setattr(bpo.repo, "_build", build_dummy) monkeypatch.setattr(bpo.images.queue, "fill", bpo_test.nop) bpo.images.queue.timer_iterate(next_interval=0.01, repo_build=False) time.sleep(0.02) bpo.repo.build() time.sleep(0.02) bpo.repo.build() assert threads == threads_expected bpo.images.queue.timer_stop()
def test_public_update_job_status(monkeypatch): arch = "x86_64" branch = "master" pkgname = "hello-world" pkgname2 = "second-package" version = "1-r4" # Disable retry_count code path (tested separately) monkeypatch.setattr(bpo.config.const, "retry_count_max", 0) # Expect the bpo server to build "second-package" and exit the bpo server # as soon as it tries to do that def fake_build_package(arch, pkgname, branch): if pkgname == "second-package": print("bpo server tries to build expected package") bpo_test.stop_server() else: print("bpo server tries to build something else: " + str(pkgname)) bpo_test.stop_server_nok() # Fake job ID return 1337 monkeypatch.setattr(bpo.jobs.build_package, "run", fake_build_package) with bpo_test.BPOServer(): # Add "hello-world" package to DB, status: building session = bpo.db.session() package = bpo.db.Package(arch, branch, pkgname, version) package.status = bpo.db.PackageStatus.building package.job_id = 1111 session.merge(package) # Add "second-package" to DB, status: queued package2 = bpo.db.Package(arch, branch, pkgname2, version) session.merge(package2) session.commit() # Call update-job-status to set "hello-world" status to "failed" and # to start building "second-package" requests.post("http://127.0.0.1:5000/api/public/update-job-status") # Verify package status bpo_test.assert_package(pkgname, status="failed")
def test_callback_depends_remove_deleted_packages_db(monkeypatch): # Stop bpo server after bpo.repo.build was called 3x global stop_count stop_count = 0 def stop_count_increase(*args, **kwargs): global stop_count stop_count += 1 print("stop_count_increase: " + str(stop_count)) if stop_count == 3: bpo_test.stop_server() monkeypatch.setattr(bpo.repo, "build", stop_count_increase) # Fill the db with "hello-world", "hello-world-wrapper" with bpo_test.BPOServer(): bpo_test.trigger.job_callback_get_depends("master") # Insert a new package, that does not exist in the depends payload session = bpo.db.session() arch = "x86_64" branch = "master" pkgname = "pkg-not-in-payload" version = "1337-r42" package_db = bpo.db.Package(arch, branch, pkgname, version) session.merge(package_db) session.commit() # Put fake apk with a valid name for the new db entry in final repo final_path = bpo.repo.final.get_path(arch, branch) apk_path = "{}/{}-{}.apk".format(final_path, pkgname, version) os.makedirs(final_path) shutil.copy(__file__, apk_path) # Indirectly trigger bpo.get_depends.remove_deleted_packages_db() bpo_test.trigger.job_callback_get_depends("master") # Package must not exist in db anymore (it isn't in the payload) # (apk still exists, because bpo.repo.build was monkeypatched) assert bpo.db.get_package(session, pkgname, arch, branch) is None
def test_repo_next_package_to_build(monkeypatch): # Disable retry_count code path (tested separately) monkeypatch.setattr(bpo.config.const, "retry_count_max", 0) # Fill the db with "hello-world", "hello-world-wrapper" with bpo_test.BPOServer(): monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.trigger.job_callback_get_depends("master") session = bpo.db.session() func = bpo.repo.next_package_to_build arch = "x86_64" branch = "master" # First package should be "hello-world" assert func(session, arch, branch) == "hello-world" # Change "hello-world" to failed package = bpo.db.get_package(session, "hello-world", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.failed) # Remaining "hello-world-wrapper" depends on failing package "hello-world" assert func(session, arch, branch) is None
def test_timer(monkeypatch): global fake_fill_i fake_fill_i = 0 def fake_fill(): global fake_fill_i fake_fill_i += 1 print(f"fake_fill_i: {fake_fill_i}") # Clear the database monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.BPOServer() # Run timer_iterate with a tiny interval monkeypatch.setattr(bpo.repo, "build", bpo_test.nop) monkeypatch.setattr(bpo.images.queue, "fill", fake_fill) bpo.images.queue.timer_iterate(0.01, False) # Let it run bpo.images.queue.fill() three times while fake_fill_i != 3: time.sleep(0.01) bpo.images.queue.timer_stop()
def test_build_package_run_skip_existing(monkeypatch): # Fill the db with "hello-world", "hello-world-wrapper" with bpo_test.BPOServer(): monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.trigger.job_callback_get_depends("master") # Package status should be "queued" session = bpo.db.session() pkgname = "hello-world" arch = "x86_64" branch = "master" package = bpo.db.get_package(session, pkgname, arch, branch) assert package.status == bpo.db.PackageStatus.queued # Copy hello-world apk to wip repo testdata_path = bpo.config.const.top_dir + "/test/testdata/" apk_hello = testdata_path + "/hello-world-1-r4.apk" wip_path = bpo.repo.wip.get_path(arch, branch) os.makedirs(wip_path) shutil.copy(apk_hello, wip_path) # Build should be skipped assert bpo.jobs.build_package.run(arch, pkgname, branch) is False bpo_test.assert_package(pkgname, status="built")
def test_build_final_repo_with_two_pkgs_SLOW_120s(monkeypatch, tmpdir): # Prepare job-callback/get-depends payload payload_path = str(tmpdir) + "/payload.json" v_hello = bpo_test.const.version_hello_world v_wrapper = bpo_test.const.version_hello_world_wrapper overrides = { "hello-world": { "version": v_hello }, "hello-world-wrapper": { "version": v_wrapper } } bpo_test.trigger.override_depends_json(payload_path, overrides) with bpo_test.BPOServer(): # Trigger job-callback/get-depends and let it run all the way until the # final repository is ready to be published monkeypatch.setattr(bpo.repo.final, "publish", bpo_test.stop_server) bpo_test.trigger.job_callback_get_depends("master", payload_path=payload_path) # WIP repo must be empty arch = "x86_64" branch = "master" path = bpo.repo.wip.get_path(arch, branch) apks = bpo.repo.get_apks(path) assert apks == [] # Final repo must have both packages path = bpo.repo.final.get_path(arch, branch) apks = bpo.repo.get_apks(path) assert apks == [ "hello-world-" + v_hello + ".apk", "hello-world-wrapper-" + v_wrapper + ".apk" ]
def test_build_arch_branch(monkeypatch): """ Test all code paths of bpo.repo.build_arch_branch(). Create the usual test database with the two hello-world and hello-world-wrapper packages, then let it build one package after another. When called again, it must figure out, that the WIP repo is complete and start to create the symlink repo. Also test the case, where the repo is stuck (hello-world failed, but -wrapper depends on it and can't be built). """ # Disable retry_count code path (tested separately) monkeypatch.setattr(bpo.config.const, "retry_count_max", 0) # *** Monkeypatch functions *** # bpo.jobs.build_package.run global build_package_run_called global expected_pkgname def build_package_run(arch, pkgname, branch): global build_package_run_called global expected_pkgname build_package_run_called = True assert pkgname == expected_pkgname return True monkeypatch.setattr(bpo.jobs.build_package, "run", build_package_run) # bpo.repo.set_stuck global build_repo_stuck build_repo_stuck = False def bpo_repo_set_stuck(arch, branch): global build_repo_stuck build_repo_stuck = True monkeypatch.setattr(bpo.repo, "set_stuck", bpo_repo_set_stuck) # bpo.repo.symlink.create global bpo_symlink_create_called def bpo_repo_symlink_create(arch, branch, force): global bpo_symlink_create_called bpo_symlink_create_called = True monkeypatch.setattr(bpo.repo.symlink, "create", bpo_repo_symlink_create) # *** Prepare test *** # Fill the db with "hello-world", "hello-world-wrapper" with bpo_test.BPOServer(): monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.trigger.job_callback_get_depends("master") # Function and arguments variables session = bpo.db.session() slots_available = 1 arch = "x86_64" branch = "master" func = bpo.repo.build_arch_branch # *** Test building all packages successfully *** # Start building "hello-world" (1/2) build_package_run_called = False expected_pkgname = "hello-world" assert func(session, slots_available, arch, branch) == 1 assert build_package_run_called assert build_repo_stuck is False # Change "hello-world" to built package = bpo.db.get_package(session, "hello-world", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.built) # Start building "hello-world-wrapper" (2/2) build_package_run_called = False expected_pkgname = "hello-world-wrapper" assert func(session, slots_available, arch, branch) == 1 assert build_package_run_called assert build_repo_stuck is False # Change "hello-world-wrapper" to built (all packages are built!) package = bpo.db.get_package(session, "hello-world-wrapper", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.built) # Create symlink repo build_package_run_called = False bpo_symlink_create_called = False assert func(session, slots_available, arch, branch) == 0 assert build_package_run_called is False assert bpo_symlink_create_called assert build_repo_stuck is False # *** Test repo being stuck *** # Change "hello-world" to failed package = bpo.db.get_package(session, "hello-world", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.failed) # Change "hello-world-wrapper" to queued package = bpo.db.get_package(session, "hello-world-wrapper", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.queued) # Expect build_repo_stuck log message build_package_run_called = False bpo_symlink_create_called = False assert func(session, slots_available, arch, branch) == 0 assert build_package_run_called is False assert bpo_symlink_create_called is False assert build_repo_stuck is True # *** Test repo being stuck (depend is building) *** # Change "hello-world" to building package = bpo.db.get_package(session, "hello-world", arch, branch) bpo.db.set_package_status(session, package, bpo.db.PackageStatus.building) # Expect build_repo_stuck log message build_package_run_called = False bpo_symlink_create_called = False assert func(session, slots_available, arch, branch) == 0 assert build_package_run_called is False assert bpo_symlink_create_called is False assert build_repo_stuck is True
def test_callback_depends_to_nop(monkeypatch): with bpo_test.BPOServer(): # Trigger job-callback/get-depends monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server) bpo_test.trigger.job_callback_get_depends("master")