def test_remove_old(monkeypatch):
monkeypatch.setattr(bpo.config.const.images, "images", {
"qemu-amd64": {
"branches": ["master"],
"branch_configs": {
"master": {"ui": ["sxmo"],
"keep": 2}
}
}})
branch = "master"
device = "qemu-amd64"
ui = "sxmo"
# Init and clear DB
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.BPOServer()
session = bpo.db.session()
# Add four test images (similar to job_callback_build_image())
for i in range(1, 5):
date = datetime.datetime.fromisoformat(f"2021-01-0{i}")
dir_name = f"2021010{i}-1337"
# Add to DB
image = bpo.db.Image(device, branch, ui)
bpo.db.set_image_status(session, image, bpo.db.ImageStatus.published,
job_id=i, dir_name=dir_name, date=date)
# Create dir with index.html
path_img = bpo.images.path_db_obj(image)
os.makedirs(path_img)
bpo.ui.images.write_index_file_list(path_img, "05_files.html")
# Verify that image dirs were created
dir_ui = f"{bpo.config.args.images_path}/edge/qemu-amd64/sxmo"
assert os.path.exists(f"{dir_ui}/20210101-1337/index.html")
assert os.path.exists(f"{dir_ui}/20210102-1337/index.html")
assert os.path.exists(f"{dir_ui}/20210103-1337/index.html")
assert os.path.exists(f"{dir_ui}/20210104-1337/index.html")
# Run remove_old with keep=2
bpo.images.remove_old()
# Verify that the two oldest image dir were removed
assert not os.path.exists(f"{dir_ui}/20210101-1337")
assert not os.path.exists(f"{dir_ui}/20210102-1337")
assert os.path.exists(f"{dir_ui}/20210103-1337/index.html")
assert os.path.exists(f"{dir_ui}/20210104-1337/index.html")
def test_depends_SLOW_60s(monkeypatch):
""" Trigger the api push hook, then let bpo run the depends job.
Monkeypatch bpo.repo.build, so it stops after receiving depends
and does not try to build the repo. """
# Limit to two arches (more would increase test time)
branches = collections.OrderedDict()
branches["master"] = {
"arches": ["x86_64", "armv7"],
"ignore_errors": False
}
monkeypatch.setattr(bpo.config.const, "branches", branches)
with bpo_test.BPOServer():
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.trigger.push_hook_gitlab()
def test_build_thread_safety(monkeypatch):
"""
Verify that bpo.repo.build doesn't run more than once at a time, even if
both MainThread and ImageTimerThread happen to call it at the same time.
Expected timeline:
ImageTimerThread |..|bb|bb|bb|..|ww|ww|bb|bb|bb|..|ww|ww|
MainThread |..|..|ww|ww|bb|bb|bb|..|..|ww|bb|bb|bb|
|--|--|--|--|--|--|--|--|--|--|--|--|--|>time in 0.01s
00 01 02 03 04 05 06 07 08 09 10 11 12 13
|..|: time.sleep in MainThread / timer interval sleep in ImageTimerThread
|bb|: running bpo.repo.build (build_dummy below)
|ww|: waiting for bpo.repo.build due to lock
If the lock did not work, the ImageTimerThread would run not just two, but
three times (take out the |ww| blocks). Therefore the test shows that the
function is thread safe.
"""
bpo_test.BPOServer()
# The order in which the threads run bpo.repo.build
threads_expected = [
"ImageTimerThread", "MainThread", "ImageTimerThread", "MainThread"
]
threads = []
def build_dummy(force_repo_update=False, no_repo_update=False):
logging.info("build_dummy called")
threads.append(threading.current_thread().name)
time.sleep(0.03)
monkeypatch.setattr(bpo.repo, "_build", build_dummy)
monkeypatch.setattr(bpo.images.queue, "fill", bpo_test.nop)
bpo.images.queue.timer_iterate(next_interval=0.01, repo_build=False)
time.sleep(0.02)
bpo.repo.build()
time.sleep(0.02)
bpo.repo.build()
assert threads == threads_expected
bpo.images.queue.timer_stop()
def test_public_update_job_status(monkeypatch):
arch = "x86_64"
branch = "master"
pkgname = "hello-world"
pkgname2 = "second-package"
version = "1-r4"
# Disable retry_count code path (tested separately)
monkeypatch.setattr(bpo.config.const, "retry_count_max", 0)
# Expect the bpo server to build "second-package" and exit the bpo server
# as soon as it tries to do that
def fake_build_package(arch, pkgname, branch):
if pkgname == "second-package":
print("bpo server tries to build expected package")
bpo_test.stop_server()
else:
print("bpo server tries to build something else: " + str(pkgname))
bpo_test.stop_server_nok()
# Fake job ID
return 1337
monkeypatch.setattr(bpo.jobs.build_package, "run", fake_build_package)
with bpo_test.BPOServer():
# Add "hello-world" package to DB, status: building
session = bpo.db.session()
package = bpo.db.Package(arch, branch, pkgname, version)
package.status = bpo.db.PackageStatus.building
package.job_id = 1111
session.merge(package)
# Add "second-package" to DB, status: queued
package2 = bpo.db.Package(arch, branch, pkgname2, version)
session.merge(package2)
session.commit()
# Call update-job-status to set "hello-world" status to "failed" and
# to start building "second-package"
requests.post("http://127.0.0.1:5000/api/public/update-job-status")
# Verify package status
bpo_test.assert_package(pkgname, status="failed")
def test_callback_depends_remove_deleted_packages_db(monkeypatch):
# Stop bpo server after bpo.repo.build was called 3x
global stop_count
stop_count = 0
def stop_count_increase(*args, **kwargs):
global stop_count
stop_count += 1
print("stop_count_increase: " + str(stop_count))
if stop_count == 3:
bpo_test.stop_server()
monkeypatch.setattr(bpo.repo, "build", stop_count_increase)
# Fill the db with "hello-world", "hello-world-wrapper"
with bpo_test.BPOServer():
bpo_test.trigger.job_callback_get_depends("master")
# Insert a new package, that does not exist in the depends payload
session = bpo.db.session()
arch = "x86_64"
branch = "master"
pkgname = "pkg-not-in-payload"
version = "1337-r42"
package_db = bpo.db.Package(arch, branch, pkgname, version)
session.merge(package_db)
session.commit()
# Put fake apk with a valid name for the new db entry in final repo
final_path = bpo.repo.final.get_path(arch, branch)
apk_path = "{}/{}-{}.apk".format(final_path, pkgname, version)
os.makedirs(final_path)
shutil.copy(__file__, apk_path)
# Indirectly trigger bpo.get_depends.remove_deleted_packages_db()
bpo_test.trigger.job_callback_get_depends("master")
# Package must not exist in db anymore (it isn't in the payload)
# (apk still exists, because bpo.repo.build was monkeypatched)
assert bpo.db.get_package(session, pkgname, arch, branch) is None
def test_repo_next_package_to_build(monkeypatch):
# Disable retry_count code path (tested separately)
monkeypatch.setattr(bpo.config.const, "retry_count_max", 0)
# Fill the db with "hello-world", "hello-world-wrapper"
with bpo_test.BPOServer():
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.trigger.job_callback_get_depends("master")
session = bpo.db.session()
func = bpo.repo.next_package_to_build
arch = "x86_64"
branch = "master"
# First package should be "hello-world"
assert func(session, arch, branch) == "hello-world"
# Change "hello-world" to failed
package = bpo.db.get_package(session, "hello-world", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.failed)
# Remaining "hello-world-wrapper" depends on failing package "hello-world"
assert func(session, arch, branch) is None
def test_timer(monkeypatch):
global fake_fill_i
fake_fill_i = 0
def fake_fill():
global fake_fill_i
fake_fill_i += 1
print(f"fake_fill_i: {fake_fill_i}")
# Clear the database
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.BPOServer()
# Run timer_iterate with a tiny interval
monkeypatch.setattr(bpo.repo, "build", bpo_test.nop)
monkeypatch.setattr(bpo.images.queue, "fill", fake_fill)
bpo.images.queue.timer_iterate(0.01, False)
# Let it run bpo.images.queue.fill() three times
while fake_fill_i != 3:
time.sleep(0.01)
bpo.images.queue.timer_stop()
def test_build_package_run_skip_existing(monkeypatch):
# Fill the db with "hello-world", "hello-world-wrapper"
with bpo_test.BPOServer():
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.trigger.job_callback_get_depends("master")
# Package status should be "queued"
session = bpo.db.session()
pkgname = "hello-world"
arch = "x86_64"
branch = "master"
package = bpo.db.get_package(session, pkgname, arch, branch)
assert package.status == bpo.db.PackageStatus.queued
# Copy hello-world apk to wip repo
testdata_path = bpo.config.const.top_dir + "/test/testdata/"
apk_hello = testdata_path + "/hello-world-1-r4.apk"
wip_path = bpo.repo.wip.get_path(arch, branch)
os.makedirs(wip_path)
shutil.copy(apk_hello, wip_path)
# Build should be skipped
assert bpo.jobs.build_package.run(arch, pkgname, branch) is False
bpo_test.assert_package(pkgname, status="built")
def test_build_final_repo_with_two_pkgs_SLOW_120s(monkeypatch, tmpdir):
# Prepare job-callback/get-depends payload
payload_path = str(tmpdir) + "/payload.json"
v_hello = bpo_test.const.version_hello_world
v_wrapper = bpo_test.const.version_hello_world_wrapper
overrides = {
"hello-world": {
"version": v_hello
},
"hello-world-wrapper": {
"version": v_wrapper
}
}
bpo_test.trigger.override_depends_json(payload_path, overrides)
with bpo_test.BPOServer():
# Trigger job-callback/get-depends and let it run all the way until the
# final repository is ready to be published
monkeypatch.setattr(bpo.repo.final, "publish", bpo_test.stop_server)
bpo_test.trigger.job_callback_get_depends("master",
payload_path=payload_path)
# WIP repo must be empty
arch = "x86_64"
branch = "master"
path = bpo.repo.wip.get_path(arch, branch)
apks = bpo.repo.get_apks(path)
assert apks == []
# Final repo must have both packages
path = bpo.repo.final.get_path(arch, branch)
apks = bpo.repo.get_apks(path)
assert apks == [
"hello-world-" + v_hello + ".apk",
"hello-world-wrapper-" + v_wrapper + ".apk"
]
def test_build_arch_branch(monkeypatch):
""" Test all code paths of bpo.repo.build_arch_branch(). Create the usual
test database with the two hello-world and hello-world-wrapper
packages, then let it build one package after another. When called
again, it must figure out, that the WIP repo is complete and start to
create the symlink repo. Also test the case, where the repo is stuck
(hello-world failed, but -wrapper depends on it and can't be built).
"""
# Disable retry_count code path (tested separately)
monkeypatch.setattr(bpo.config.const, "retry_count_max", 0)
# *** Monkeypatch functions ***
# bpo.jobs.build_package.run
global build_package_run_called
global expected_pkgname
def build_package_run(arch, pkgname, branch):
global build_package_run_called
global expected_pkgname
build_package_run_called = True
assert pkgname == expected_pkgname
return True
monkeypatch.setattr(bpo.jobs.build_package, "run", build_package_run)
# bpo.repo.set_stuck
global build_repo_stuck
build_repo_stuck = False
def bpo_repo_set_stuck(arch, branch):
global build_repo_stuck
build_repo_stuck = True
monkeypatch.setattr(bpo.repo, "set_stuck", bpo_repo_set_stuck)
# bpo.repo.symlink.create
global bpo_symlink_create_called
def bpo_repo_symlink_create(arch, branch, force):
global bpo_symlink_create_called
bpo_symlink_create_called = True
monkeypatch.setattr(bpo.repo.symlink, "create", bpo_repo_symlink_create)
# *** Prepare test ***
# Fill the db with "hello-world", "hello-world-wrapper"
with bpo_test.BPOServer():
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.trigger.job_callback_get_depends("master")
# Function and arguments variables
session = bpo.db.session()
slots_available = 1
arch = "x86_64"
branch = "master"
func = bpo.repo.build_arch_branch
# *** Test building all packages successfully ***
# Start building "hello-world" (1/2)
build_package_run_called = False
expected_pkgname = "hello-world"
assert func(session, slots_available, arch, branch) == 1
assert build_package_run_called
assert build_repo_stuck is False
# Change "hello-world" to built
package = bpo.db.get_package(session, "hello-world", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.built)
# Start building "hello-world-wrapper" (2/2)
build_package_run_called = False
expected_pkgname = "hello-world-wrapper"
assert func(session, slots_available, arch, branch) == 1
assert build_package_run_called
assert build_repo_stuck is False
# Change "hello-world-wrapper" to built (all packages are built!)
package = bpo.db.get_package(session, "hello-world-wrapper", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.built)
# Create symlink repo
build_package_run_called = False
bpo_symlink_create_called = False
assert func(session, slots_available, arch, branch) == 0
assert build_package_run_called is False
assert bpo_symlink_create_called
assert build_repo_stuck is False
# *** Test repo being stuck ***
# Change "hello-world" to failed
package = bpo.db.get_package(session, "hello-world", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.failed)
# Change "hello-world-wrapper" to queued
package = bpo.db.get_package(session, "hello-world-wrapper", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.queued)
# Expect build_repo_stuck log message
build_package_run_called = False
bpo_symlink_create_called = False
assert func(session, slots_available, arch, branch) == 0
assert build_package_run_called is False
assert bpo_symlink_create_called is False
assert build_repo_stuck is True
# *** Test repo being stuck (depend is building) ***
# Change "hello-world" to building
package = bpo.db.get_package(session, "hello-world", arch, branch)
bpo.db.set_package_status(session, package, bpo.db.PackageStatus.building)
# Expect build_repo_stuck log message
build_package_run_called = False
bpo_symlink_create_called = False
assert func(session, slots_available, arch, branch) == 0
assert build_package_run_called is False
assert bpo_symlink_create_called is False
assert build_repo_stuck is True
def test_callback_depends_to_nop(monkeypatch):
with bpo_test.BPOServer():
# Trigger job-callback/get-depends
monkeypatch.setattr(bpo.repo, "build", bpo_test.stop_server)
bpo_test.trigger.job_callback_get_depends("master")
