def test_fetch_and_reset(temp_repo_static): """ Tests updating already cloned repo, which was rebased. Prevents ``fatal: refusing to merge unrelated histories``. """ arca = Arca(base_dir=BASE_DIR) initial_value = str(uuid4()) temp_repo_static.file_path.write_text(initial_value) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Update") initial_commit = temp_repo_static.repo.head.object.hexsha for _ in range(5): temp_repo_static.file_path.write_text(str(uuid4())) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Update") arca.get_files(temp_repo_static.url, temp_repo_static.branch) temp_repo_static.repo.head.reset(initial_commit) _, path_to_cloned = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert (path_to_cloned / temp_repo_static.file_path.name).read_text() == initial_value
def test_inherit_image(temp_repo_func): backend = DockerBackend(verbosity=2, inherit_image="python:alpine3.6") arca = Arca(backend=backend, base_dir=BASE_DIR) task = Task("test_file:return_str_function") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" temp_repo_func.file_path.write_text(RETURN_PLATFORM) temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Platform") task = Task("test_file:return_platform") # debian is the default, alpine dist() returns ('', '', '') on - so this fails when the default image is used assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output != "debian" requirements_path = temp_repo_func.repo_path / backend.requirements_location requirements_path.write_text("colorama==0.3.9") temp_repo_func.file_path.write_text(RETURN_COLORAMA_VERSION_FUNCTION) temp_repo_func.repo.index.add( [str(temp_repo_func.file_path), str(requirements_path)]) temp_repo_func.repo.index.commit("Added requirements, changed to version") colorama_task = Task("test_file:return_str_function") assert arca.run(temp_repo_func.url, temp_repo_func.branch, colorama_task).output == "0.3.9"
def test_invalid_arguments(): with pytest.raises(ArcaMisconfigured): Arca(base_dir=BASE_DIR, single_pull=True, settings={ "ARCA_CACHE_BACKEND": "dogpile.cache.dbm", "ARCA_CACHE_BACKEND_ARGUMENTS": json.dumps( {"filename": str(Path(BASE_DIR) / "cachefile.dbm")})[:-1] }) # in case ignore is set, no error thrown, region configured arca = Arca(base_dir=BASE_DIR, single_pull=True, ignore_cache_errors=True, settings={ "ARCA_CACHE_BACKEND": "dogpile.cache.dbm", "ARCA_CACHE_BACKEND_ARGUMENTS": json.dumps( {"filename": str(Path(BASE_DIR) / "cachefile.dbm")})[:-1] }) assert arca.region.is_configured
def test_repo_id(url): arca = Arca() repo_id = arca.repo_id(url) # it's a valid folder name with only alphanumeric, dot or underscore characters assert re.match(r"^[a-zA-Z0-9._]+$", repo_id)
def test_repo_id_unique(): arca = Arca() repo_id_1 = arca.repo_id("http://github.com/pyvec/naucse.python.cz") repo_id_2 = arca.repo_id("http://github.com_pyvec_naucse.python.cz") assert repo_id_1 != repo_id_2
def test_keep_container_running(temp_repo_func): backend = DockerBackend(verbosity=2, keep_container_running=True) arca = Arca(backend=backend, base_dir=BASE_DIR) task = Task("test_file:return_str_function") backend.check_docker_access() # init docker client container_count = len(backend.client.containers.list()) assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" count_after_run = len(backend.client.containers.list()) assert count_after_run == container_count + 1 # let's assume no changes are done to containers elsewhere assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" count_after_second_run = len(backend.client.containers.list()) assert count_after_second_run == container_count + 1 # the count is still the same backend.stop_containers() count_after_stop = len(backend.client.containers.list()) assert count_after_stop == container_count
def test_unicode_path(temp_repo_func,): backend = CurrentEnvironmentBackend(verbosity=2) arca = Arca(backend=backend, base_dir=BASE_DIR + "/abčď", single_pull=True) task = Task("test_file:return_str_function") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string"
def test_python_version(temp_repo_func, python_version): backend = DockerBackend(verbosity=2, python_version=python_version) arca = Arca(backend=backend, base_dir=BASE_DIR) temp_repo_func.file_path.write_text(RETURN_PYTHON_VERSION_FUNCTION) temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Initial") task = Task("test_file:return_python_version") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == python_version
def test_reference(): arca = Arca(base_dir=BASE_DIR) branch = "master" git_dir_1 = Path("/tmp/arca/") / str(uuid4()) git_url_1 = f"file://{git_dir_1}" filepath_1 = git_dir_1 / "test_file.txt" repo_1 = Repo.init(git_dir_1) last_uuid = None for _ in range(20): last_uuid = str(uuid4()) filepath_1.write_text(last_uuid) repo_1.index.add([str(filepath_1)]) repo_1.index.commit("Initial") # test nonexistent reference cloned_repo, cloned_repo_path = arca.get_files(git_url_1, branch, reference=Path("/tmp/arca/") / str(uuid4())) assert (cloned_repo_path / "test_file.txt").read_text() == last_uuid shutil.rmtree(str(cloned_repo_path)) # test existing reference with no common commits git_dir_2 = Path("/tmp/arca/") / str(uuid4()) filepath_2 = git_dir_2 / "test_file.txt" repo_2 = Repo.init(git_dir_2) for _ in range(20): filepath_2.write_text(str(uuid4())) repo_2.index.add([str(filepath_2)]) repo_2.index.commit("Initial") cloned_repo, cloned_repo_path = arca.get_files(git_url_1, branch, reference=git_dir_2) assert (cloned_repo_path / "test_file.txt").read_text() == last_uuid shutil.rmtree(str(cloned_repo_path)) # test existing reference with common commits git_dir_3 = Path("/tmp/arca/") / str(uuid4()) git_url_3 = f"file://{git_dir_3}" filepath_3 = git_dir_3 / "test_file.txt" repo_3 = repo_1.clone(str(git_dir_3)) # must pass string, fails otherwise for _ in range(20): last_uuid = str(uuid4()) filepath_3.write_text(last_uuid) repo_3.index.add([str(filepath_3)]) repo_3.index.commit("Initial") cloned_repo, cloned_repo_path = arca.get_files(git_url_3, branch, reference=git_dir_1) assert (cloned_repo_path / "test_file.txt").read_text() == last_uuid
def test_output(temp_repo_func): arca = Arca(backend=CurrentEnvironmentBackend) temp_repo_func.file_path.write_text(PRINTING_FUNCTION) temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Initial") result = arca.run(temp_repo_func.url, temp_repo_func.branch, Task("test_file:func")) assert result.output == 1 assert result.stdout == "Printed to stdout\nWritten to stdout" assert result.stderr == "Printed to stderr\nWritten to stderr"
def test_static_files(temp_repo_static, file_location): arca = Arca(base_dir=BASE_DIR) filepath = temp_repo_static.file_path if file_location: new_filepath = temp_repo_static.repo_path / file_location / "test_file.txt" new_filepath.parent.mkdir(exist_ok=True, parents=True) filepath.replace(new_filepath) temp_repo_static.repo.index.add([str(new_filepath)]) temp_repo_static.repo.index.remove([str(filepath)]) temp_repo_static.repo.index.commit("Initial") filepath = new_filepath relative_path = Path(file_location) / "test_file.txt" nonexistent_relative_path = Path(file_location) / "test_file2.txt" result = arca.static_filename(temp_repo_static.url, temp_repo_static.branch, relative_path) assert filepath.read_text() == result.read_text() result = arca.static_filename(temp_repo_static.url, temp_repo_static.branch, str(relative_path)) assert filepath.read_text() == result.read_text() with pytest.raises(FileOutOfRangeError): arca.static_filename(temp_repo_static.url, temp_repo_static.branch, "../file.txt") with pytest.raises(FileNotFoundError): arca.static_filename(temp_repo_static.url, temp_repo_static.branch, nonexistent_relative_path)
def test_cache_backend_module_not_found(): # redis must not be present in the env with pytest.raises(ImportError): import redis # noqa with pytest.raises(ModuleNotFoundError): Arca(base_dir=BASE_DIR, single_pull=True, settings={"ARCA_CACHE_BACKEND": "dogpile.cache.redis"}) arca = Arca(base_dir=BASE_DIR, single_pull=True, ignore_cache_errors=True, settings={"ARCA_CACHE_BACKEND": "dogpile.cache.redis"}) assert arca.region.is_configured
def list_python_versions(self): from arca import Arca arca = Arca() _, pyenv = arca.get_files("https://github.com/pyenv/pyenv.git", "master") build_for = re.compile(r"^3\.[67]\.[0-9]+$") for path in sorted( (pyenv / "plugins/python-build/share/python-build/").iterdir()): if path.is_dir(): continue if build_for.match(path.name): yield path.name
def test_apt_dependencies(temp_repo_func): backend = DockerBackend(verbosity=2, apt_dependencies=["libasound2-dev"]) arca = Arca(backend=backend, base_dir=BASE_DIR) requirements_path = temp_repo_func.repo_path / "requirements.txt" requirements_path.write_text("pyalsaaudio==0.8.4") temp_repo_func.file_path.write_text(RETURN_ALSAAUDIO_INSTALLED) temp_repo_func.repo.index.add( [str(temp_repo_func.file_path), str(requirements_path)]) temp_repo_func.repo.index.commit("Added requirements, changed to lxml") # pyalsaaudio can't be installed if libasound2-dev is missing task = Task("test_file:return_alsaaudio_installed") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output
def test_single_pull(temp_repo_func, mocker): backend = CurrentEnvironmentBackend(verbosity=2, current_environment_requirements=None) arca = Arca(backend=backend, base_dir=BASE_DIR, single_pull=True) mocker.spy(arca, "_pull") task = Task("test_file:return_str_function") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" assert arca._pull.call_count == 1 temp_repo_func.file_path.write_text(SECOND_RETURN_STR_FUNCTION) temp_repo_func.repo.index.add([str(temp_repo_func.file_path)]) temp_repo_func.repo.index.commit("Updated function") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "Some string" assert arca._pull.call_count == 1 arca.pull_again(temp_repo_func.url, temp_repo_func.branch) assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == TEST_UNICODE assert arca._pull.call_count == 2
def test_depth(temp_repo_static): arca = Arca(base_dir=BASE_DIR) for _ in range(19): # since one commit is made in the fixture temp_repo_static.file_path.write_text(str(uuid4())) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Initial") # test that in default settings, the whole repo is pulled in one go cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert cloned_repo.commit().count() == 1 # default is 1 # test when pulled again, the depth is increased since the local copy is stored temp_repo_static.file_path.write_text(str(uuid4())) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Initial") cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert cloned_repo.commit().count() == 2 shutil.rmtree(str(cloned_repo_path)) # test that when setting a certain depth, at least the depth is pulled (in case of merges etc) cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch, depth=10) assert cloned_repo.commit().count() >= 10 before_second_pull = cloned_repo.commit().count() # test when pulled again, the depth setting is ignored temp_repo_static.file_path.write_text(str(uuid4())) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Initial") cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch) assert cloned_repo.commit().count() == before_second_pull + 1 shutil.rmtree(str(cloned_repo_path)) # test when setting depth bigger than repo size, no fictional commits are included cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch, depth=100) assert cloned_repo.commit().count() == 22 # 20 plus the 2 extra commits shutil.rmtree(str(cloned_repo_path)) # test no limit cloned_repo, cloned_repo_path = arca.get_files(temp_repo_static.url, temp_repo_static.branch, depth=None) assert cloned_repo.commit().count() == 22 # 20 plus the 2 extra commits
def test_requirements_strategy(): """ Test validation of invalid strategies """ with pytest.raises(ValueError): Arca(backend=CurrentEnvironmentBackend( verbosity=2, current_environment_requirements=None, requirements_strategy="nonexistant_strategy" ), base_dir=BASE_DIR) with pytest.raises(ValueError): arca = Arca(base_dir=BASE_DIR, settings={ "ARCA_BACKEND": "arca.backend.CurrentEnvironmentBackend", "ARCA_BACKEND_VERBOSITY": 2, "ARCA_BACKEND_CURRENT_ENVIRONMENT_REQUIREMENTS": None, "ARCA_BACKEND_REQUIREMENTS_STRATEGY": "nonexistant_strategy" }) print(arca.backend.requirements_strategy)
def test_environ(): os.environ["ARCA_TEST_ONE"] = "test" os.environ["ARCA_TEST_THREE"] = "test3" os.environ["ACRA_TEST_FOUR"] = "test4" arca = Arca(settings={"ARCA_TEST_ONE": 1, "ARCA_TEST_TWO": 2}) assert arca.settings.get("test_one") == "test" assert arca.settings.get("test_three") == "test3" with pytest.raises(KeyError): arca.settings.get("test_four")
def test_json_loads_arguments(): arca = Arca(base_dir=BASE_DIR, settings={ "ARCA_CACHE_BACKEND": "dogpile.cache.dbm", "ARCA_CACHE_BACKEND_ARGUMENTS": json.dumps( {"filename": str(Path(BASE_DIR) / "cachefile.dbm")}) }) assert arca.region.is_configured
def test_push_to_registry(temp_repo_func, mocker): backend = DockerBackend(verbosity=2, use_registry_name="docker.io/mikicz/arca-test") arca = Arca(backend=backend, base_dir=BASE_DIR) temp_repo_func.file_path.write_text(RETURN_COLORAMA_VERSION_FUNCTION) requirements_path = temp_repo_func.repo_path / backend.requirements_location requirements_path.write_text("colorama==0.3.9") temp_repo_func.repo.index.add( [str(temp_repo_func.file_path), str(requirements_path)]) temp_repo_func.repo.index.commit("Initial") task = Task("test_file:return_str_function") # even though the image might already exist on the registry, lets pretend it doesn't mocker.patch.object(backend, "try_pull_image_from_registry", lambda *args: None) assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "0.3.9" mocker.stopall() image = backend.get_image_for_repo(temp_repo_func.url, temp_repo_func.branch, temp_repo_func.repo, temp_repo_func.repo_path) backend.client.images.remove(image.id, force=True) mocker.spy(backend, "build_image") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "0.3.9" assert backend.build_image.call_count == 0 # test push disabled mocker.patch.object(backend, "try_pull_image_from_registry", lambda *args: None) # untag the image so Arca thinks the images was just built and that it needs to be pushed for image in backend.client.images.list("docker.io/mikicz/arca-test"): for tag in image.tags: if tag.startswith("docker.io/mikicz/arca-test"): backend.client.images.remove(tag) backend = DockerBackend(verbosity=2, use_registry_name="docker.io/mikicz/arca-test", registry_pull_only=True) arca = Arca(backend=backend, base_dir=BASE_DIR) mocker.spy(backend, "push_to_registry") assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "0.3.9" assert backend.push_to_registry.call_count == 0
def test_push_to_registry_fail(temp_repo_func): # when a unused repository name is used, it's created -> different username has to be used backend = DockerBackend( verbosity=2, use_registry_name="docker.io/mikicz-unknown-user/arca-test") arca = Arca(backend=backend, base_dir=BASE_DIR) temp_repo_func.file_path.write_text(RETURN_COLORAMA_VERSION_FUNCTION) requirements_path = temp_repo_func.repo_path / backend.requirements_location requirements_path.write_text("colorama==0.3.9") temp_repo_func.repo.index.add( [str(temp_repo_func.file_path), str(requirements_path)]) temp_repo_func.repo.index.commit("Initial") task = Task("test_file:return_str_function") with pytest.raises(PushToRegistryError): assert arca.run(temp_repo_func.url, temp_repo_func.branch, task).output == "0.3.9"
def test_setting_integration(): arca = Arca( settings={ "ARCA_BACKEND": "arca.backend.CurrentEnvironmentBackend", "ARCA_CURRENT_ENVIRONMENT_BACKEND_REQUIREMENTS_STRATEGY": "ignore", "ARCA_BACKEND_REQUIREMENTS_STRATEGY": "install_extra", "ARCA_BACKEND_CWD": "test/" }) assert arca.backend.requirements_strategy == RequirementsStrategy.IGNORE # specific backend setting assert arca.backend.cwd == "test/" # tests generic BACKEND settings assert arca.backend.requirements_location == "requirements.txt" # tests default value
def test_setting_integration(): arca = Arca( settings={ "ARCA_BACKEND": "arca.backend.CurrentEnvironmentBackend", "ARCA_CURRENT_ENVIRONMENT_BACKEND_REQUIREMENTS_TIMEOUT": 500, "ARCA_BACKEND_REQUIREMENTS_TIMEOUT": 600, "ARCA_BACKEND_CWD": "test/" }) assert arca.backend.requirements_timeout == 500 assert arca.backend.cwd == "test/" # tests generic BACKEND settings assert arca.backend.requirements_location == "requirements.txt" # tests default value
def init_model(): trusted = os.environ.get('NAUCSE_TRUSTED_REPOS', None) if trusted is None: trusted_repo_patterns = () else: trusted_repo_patterns = tuple(line for line in trusted.split() if line) return models.Root( url_factories={ 'api': { models.Root: lambda **kw: url_for('api', **kw), models.Course: lambda **kw: url_for('course_api', **kw), models.RunYear: lambda **kw: url_for('run_year_api', **kw), }, 'web': { models.Lesson: lambda **kw: url_for('page', page_slug='index', **kw), models.Page: lambda **kw: url_for('page', **kw), models.Solution: lambda **kw: url_for('solution', **kw), models.Course: lambda **kw: url_for('course', **kw), models.Session: lambda **kw: url_for('session', **kw), models.SessionPage: lambda **kw: url_for('session', **kw), models.StaticFile: lambda **kw: url_for('page_static', **kw), models.Root: lambda **kw: url_for('index', **kw) }, }, schema_url_factory=lambda m, is_input, **kw: url_for( 'schema', model_slug=m.model_slug, is_input=is_input, **kw), arca=Arca( settings={ "ARCA_BACKEND": "arca.backend.CurrentEnvironmentBackend", "ARCA_BACKEND_CURRENT_ENVIRONMENT_REQUIREMENTS": "requirements.txt", "ARCA_BACKEND_VERBOSITY": 2, "ARCA_BACKEND_KEEP_CONTAINER_RUNNING": True, "ARCA_BACKEND_USE_REGISTRY_NAME": "docker.io/naucse/naucse.python.cz", "ARCA_SINGLE_PULL": True, "ARCA_IGNORE_CACHE_ERRORS": True, "ARCA_CACHE_BACKEND": "dogpile.cache.dbm", "ARCA_CACHE_BACKEND_ARGUMENTS": { "filename": ".arca/cache/naucse.dbm" }, "ARCA_BASE_DIR": str(Path('.arca').resolve()), }), trusted_repo_patterns=trusted_repo_patterns, )
def test_get_reference_repository(temp_repo_static): """ Test that the :meth:`Arca.get_reference_repository` works when reference is not provided by the user or when branch `master` is not pulled first (as it is in other tests). """ temp_repo_static.file_path.write_text("master") temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit("Initial") for branch in "branch1", "branch2", "branch3": temp_repo_static.repo.create_head(branch) temp_repo_static.repo.branches[branch].checkout() temp_repo_static.file_path.write_text(branch) temp_repo_static.repo.index.add([str(temp_repo_static.file_path)]) temp_repo_static.repo.index.commit(branch) arca = Arca(base_dir=BASE_DIR) for branch in "branch1", "branch2", "master", "branch3": _, path = arca.get_files(temp_repo_static.url, branch) assert (path / temp_repo_static.file_path.name).read_text() == branch
def test_validate_repo_url(url, valid): arca = Arca() if valid: arca.validate_repo_url(url) else: with pytest.raises(ValueError): arca.validate_repo_url(url)
def arca_model(tmp_path, content_repo): """Return a model that loads remote content using Arca""" model = make_model( arca=Arca( settings={ "ARCA_BACKEND": "arca.backend.CurrentEnvironmentBackend", "ARCA_BACKEND_VERBOSITY": 2, "ARCA_BASE_DIR": str(tmp_path / '.arca'), }), # We only trust "master" branches trusted_repo_patterns=('*#master', ), ) return model
def test_current_git_hash(temp_repo_static): """ Test that the :meth:`Arca.current_git_hash <arca.Arca.current_git_hash>` behaves the same when ``single_pull`` is disabled or enabled. """ arca = Arca(base_dir=BASE_DIR) repo, _ = arca.get_files(temp_repo_static.url, temp_repo_static.branch) long_hash = arca.current_git_hash(temp_repo_static.url, temp_repo_static.branch, repo) short_hash = arca.current_git_hash(temp_repo_static.url, temp_repo_static.branch, repo, short=True) assert len(short_hash) < len(long_hash) assert long_hash.startswith(short_hash) arca = Arca(base_dir=BASE_DIR, single_pull=True) repo, _ = arca.get_files(temp_repo_static.url, temp_repo_static.branch) long_hash_single_pull = arca.current_git_hash(temp_repo_static.url, temp_repo_static.branch, repo) short_hash_single_pull = arca.current_git_hash(temp_repo_static.url, temp_repo_static.branch, repo, short=True) assert long_hash == long_hash_single_pull assert short_hash == short_hash_single_pull
def test_pull_error(): arca = Arca(base_dir=BASE_DIR) git_dir = Path("/tmp/arca/") / str(uuid4()) git_url = f"file://{git_dir}" with pytest.raises(PullError): arca.get_files(git_url, "master") filepath = git_dir / "test_file.txt" repo = Repo.init(git_dir) filepath.write_text(str(uuid4())) repo.index.add([str(filepath)]) repo.index.commit("Initial") arca.get_files(git_url, "master") with pytest.raises(PullError): arca.get_files(git_url, "some_branch") shutil.rmtree(str(git_dir)) with pytest.raises(PullError): arca.get_files(git_url, "master")
def test_current_environment_requirements(): arca = Arca(backend=CurrentEnvironmentBackend( verbosity=2, current_environment_requirements=None ), base_dir=BASE_DIR) # test how installing nonexisting packages is handled with pytest.raises(BuildError): arca.backend.install_requirements(requirements=[str(uuid4())]) with pytest.raises(ValueError): arca.backend.install_requirements() with pytest.raises(ValueError): arca.backend.install_requirements(requirements=["colorama"], _action="remove")