def test_get_settings_from_first_dci_rhel_agent_settings_file(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v2.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "remoteci/66194b70-46c5-1707-a2bb-9dde8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", "DCI_LOCAL_REPO": "/tmp/repo5", }, ) assert settings["remoteci_id"] == "66194b70-46c5-1707-a2bb-9dde8d5d4212" _topic_equals( settings["topics"][0], { "download_everything": False, "download_folder": "/tmp/repo5", "name": "RHEL-8.1", "archs": ["x86_64", "ppc64le"], "variants": [ { "name": "AppStream", "with_debug": False }, { "name": "BaseOS", "with_debug": False }, ], }, )
def test_get_settings_local_repo_added_to_an_old_settings_file(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v4.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "", "DCI_LOCAL_REPO": "/tmp/repo4", }, ) _topic_equals( settings["topics"][0], { "download_everything": False, "download_folder": "/tmp/repo5", "name": "RHEL-8.2", "archs": ["x86_64", "ppc64le"], "variants": [ { "name": "AppStream", "with_debug": False }, { "name": "BaseOS", "with_debug": False }, ], }, )
def test_get_settings_local_repo_with_multiple_topics(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v5.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) _topic_equals( settings["topics"][0], { "download_everything": False, "download_folder": "/tmp/repo6", "name": "RHEL-7.6", "archs": ["x86_64"], "variants": [], }, ) _topic_equals( settings["topics"][1], { "download_everything": False, "download_folder": "/tmp/repo6", "name": "RHEL-8.1", "archs": ["x86_64"], "variants": [], }, )
def test_get_settings_from_dci_rhel_agent_settings_file_with_only_topic_key(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v1.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", "DCI_LOCAL_REPO": "/tmp/repo4", }, ) assert settings["remoteci_id"] == "9dd94b70-1707-46c5-a2bb-661e8d5d4212" assert settings["topics"][0] == { "variants": [], "download_everything": False, "download_folder": "/tmp/repo4", "archs": ["x86_64"], "component_id": None, "dci_key_file": ANY, "components": [], "dci_cert_file": ANY, "name": "RHEL-7", "with_debug": False, "registry": None, }
def test_get_settings_read_arguments(): settings = get_settings( sys_args=[ "RHEL-8", "/tmp/repo2", "--arch", "ppc64le", "--variant", "BaseOS" ], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) assert settings["topics"][0] == { "variants": [{ "name": "BaseOS", "with_debug": False, "with_iso": False }], "download_everything": False, "download_folder": "/tmp/repo2", "archs": ["ppc64le"], "component_id": None, "dci_key_file": ANY, "components": [], "dci_cert_file": ANY, "name": "RHEL-8", "with_debug": False, "registry": None, }
def main(): settings = get_settings(sys_args=sys.argv[1:], env_variables=dict(os.environ)) exit_if_settings_invalid(settings) keys = get_keys(settings["remoteci_id"]) if keys is None: print("Can't get certificate's keys, contact DCI administrator") sys.exit(0) cert = create_temp_file(keys["cert"]).name key = create_temp_file(keys["key"]).name return_code = 0 for topic_settings in settings["topics"]: topic_name = topic_settings["name"] job = None try: topic = api.get_topic(topic_name) if topic is None: raise ("Topic name %s not found" % topic_name) job = create_job(topic["id"]) create_tag(job["id"], "download") download_components(topic_settings, api, downloader, cert, key) create_jobstate(job["id"], "success") except Exception: print("Exception when downloading components for %s" % topic_name) if job is not None: create_jobstate(job["id"], "failure") traceback.print_exc() return_code = 1 os.unlink(cert) os.unlink(key) sys.exit(return_code)
def test_nrt_always_download_metadata(): dci_files_list = { "directories": [], "files": [ { "path": "metadata", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "rpms.json", "size": 45052, }, ], "symlinks": [], } settings = get_settings( sys_args=["RHEL-8", "/tmp", "--variant", "AppStream"])["topics"][0] expected_files_list = { "directories": [], "files": [ { "path": "metadata", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "rpms.json", "size": 45052, }, ], "symlinks": [], } assert filter_files_list(dci_files_list, settings) == expected_files_list
def test_filter_files_list_with_debug_keep_os_folder_nrt(): dci_files_list = { "directories": [], "files": [ { "path": "AppStream/x86_64/debug/tree/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.x86_64.rpm", "size": 45052, }, { "path": "AppStream/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, { "path": "CRB/ppc64le/os/repodata", "sha256": "b033f8c928c3a5ba43c0a0e87a839b5ce24698e8c0a7c9c4a33e564040231805", "name": "repomd.xml", "size": 3268, }, { "path": "CRB/ppc64le/debug/tree/repodata", "sha256": "974ebd02506a4945d4f1a7ac8ac5d5a05e675e0e4bd860de667425adf02d2570", "name": "repomd.xml", "size": 1562, }, ], "symlinks": [], } args = ["RHEL-8", "/tmp", "--arch=ppc64le", "--debug"] settings = get_settings(sys_args=args)["topics"][0] expected_files_list = { "directories": [], "files": [ { "path": "CRB/ppc64le/os/repodata", "sha256": "b033f8c928c3a5ba43c0a0e87a839b5ce24698e8c0a7c9c4a33e564040231805", "name": "repomd.xml", "size": 3268, }, { "path": "CRB/ppc64le/debug/tree/repodata", "sha256": "974ebd02506a4945d4f1a7ac8ac5d5a05e675e0e4bd860de667425adf02d2570", "name": "repomd.xml", "size": 1562, }, ], "symlinks": [], } assert filter_files_list(dci_files_list, settings) == expected_files_list
def main(): api.check_repo_is_accessible() settings = get_settings(sys_args=sys.argv[1:], env_variables=dict(os.environ)) exit_if_settings_invalid(settings) configure_ssl_certificates(settings) for topic_settings in settings["topics"]: download_topic(topic_settings)
def test_non_existing_variants_are_ignored(): dci_files_list = { "directories": [], "files": [ { "path": "", "sha256": "954719cab91afac5bc142656afff86e6d8e87570b035cbce65dbbb84892a40d3", "name": ".composeinfo", "size": 14496, }, { "path": "AppStream/x86_64/debug/tree/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.x86_64.rpm", "size": 45052, }, { "path": "AppStream/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, { "path": "BaseOS/x86_64/os/Packages", "sha256": "7949b18b6d359b435686f2f5781928675ec8b2872b96f0abf6ba10747f794694", "name": "avahi-libs-0.7-19.el8.i686.rpm", "size": 68920, }, { "path": "AppStream/s390x/os/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.s390x.rpm", "size": 29562, }, ], "symlinks": [], } settings = get_settings( sys_args=["RHEL-8", "/tmp", "--variant", "Server"])["topics"][0] expected_files_list = { "directories": [], "files": [{ "path": "", "sha256": "954719cab91afac5bc142656afff86e6d8e87570b035cbce65dbbb84892a40d3", "name": ".composeinfo", "size": 14496, }], "symlinks": [], } assert filter_files_list(dci_files_list, settings) == expected_files_list
def test_get_settings_read_arguments_with_component_id(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo12", "--component-id", "c1"], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) assert settings["topics"][0]["component_id"] == "c1"
def test_get_settings_set_remoteci_id_from_env_variable(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo1"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", }, ) assert settings["remoteci_id"] == "9dd94b70-1707-46c5-a2bb-661e8d5d4212"
def test_get_settings_with_jobs_key(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v3.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) _topic_equals( settings["topics"][0], { "download_everything": False, "download_folder": "/tmp/repo10", "name": "RHEL-7.6", "archs": ["x86_64", "ppc64le"], "variants": [ { "name": "Server", "with_debug": False }, { "name": "Server-SAP", "with_debug": False }, ], }, ) _topic_equals( settings["topics"][1], { "download_everything": False, "download_folder": "/tmp/repo10", "name": "RHEL-8.1", "archs": ["x86_64"], "variants": [ { "name": "AppStream", "with_debug": False }, { "name": "BaseOS", "with_debug": True }, ], }, )
def test_with_registry(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo1", "--registry", "host:port"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", "XDG_DATA_HOME": "/home/dci/.local/share", }, ) topic = settings["topics"][0] assert topic["registry"] == "host:port"
def test_filter_files_list_with_iso(): dci_files_list = { "directories": [], "files": [ { "path": "BaseOS/x86_64/debug/tree/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.x86_64.rpm", "size": 45052, }, { "path": "BaseOS/x86_64/iso", "sha256": "06fd27c0279d5b42078f7de66d056c7875d025d1eb89a29dd2777240459c1026", "name": "RHEL-8.4.0-20201020.n.2-BaseOS-x86_64-boot.iso", "size": 731906048, }, { "path": "BaseOS/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, ], "symlinks": [], } settings = get_settings( sys_args=["RHEL-8", "/tmp", "--variant", "BaseOS", "--iso" ])["topics"][0] expected_files_list = { "directories": [], "files": [ { "path": "BaseOS/x86_64/iso", "sha256": "06fd27c0279d5b42078f7de66d056c7875d025d1eb89a29dd2777240459c1026", "name": "RHEL-8.4.0-20201020.n.2-BaseOS-x86_64-boot.iso", "size": 731906048, }, { "path": "BaseOS/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, ], "symlinks": [], } assert filter_files_list(dci_files_list, settings) == expected_files_list
def test_exit_if_architecture_in_settings_invalid_for_rhel_7(): with pytest.raises(SystemExit): exit_if_settings_invalid( get_settings( sys_args=["RHEL-7", "/tmp/repo7", "--arch", "aarch64"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "jSbJwfCdIfq12gwHAAtg5JXSBTO3wj0xkG7oW3DlqyM7bXahPRrfZlqmSv3BhmAy", "DCI_CS_URL": "https://distributed-ci.io", }, ))
def test_get_settings_set_ssl_file_from_env(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo1"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", "DCI_KEY_FILE": "/etc/dci-rhel-agent/dci.key", "DCI_CERT_FILE": "/etc/dci-rhel-agent/dci.crt", }, ) topic = settings["topics"][0] assert topic["dci_key_file"] == "/etc/dci-rhel-agent/dci.key" assert topic["dci_cert_file"] == "/etc/dci-rhel-agent/dci.crt"
def test_exit_if_settings_invalid_without_download_folder(): with pytest.raises(SystemExit): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v2.yml") exit_if_settings_invalid( get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "remoteci/66194b70-46c5-1707-a2bb-9dde8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", }, ))
def test_get_settings_set_ssl_file_from_settings_file(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v6.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) topic = settings["topics"][0] assert topic["dci_key_file"] == "/etc/dci-rhel-agent/dci.key" assert topic["dci_cert_file"] == "/etc/dci-rhel-agent/dci.crt"
def test_get_settings_download_folder_overwrite_DCI_LOCAL_REPO(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "documented_settings_file.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "", "DCI_LOCAL_REPO": "/tmp/repo4", }, ) assert settings["topics"][0]["download_folder"] == "/var/www/html" assert settings["topics"][1]["download_folder"] == "/var/www/html"
def test_get_settings_set_ssl_files_to_default_if_not_setted(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo1"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "", "DCI_CS_URL": "", "XDG_DATA_HOME": "/home/dci/.local/share", }, ) topic = settings["topics"][0] assert topic[ "dci_cert_file"] == "/home/dci/.local/share/dci-downloader/dci.crt" assert topic[ "dci_key_file"] == "/home/dci/.local/share/dci-downloader/dci.key"
def test_exit_if_settings_invalid(): try: exit_if_settings_invalid( get_settings( sys_args=["RHEL-8", "/tmp/repo7", "--variant", "BaseOS"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "jSbJwfCdIfq12gwHAAtg5JXSBTO3wj0xkG7oW3DlqyM7bXahPRrfZlqmSv3BhmAy", "DCI_CS_URL": "https://distributed-ci.io", }, )) except SystemExit: pytest.fail("exit_if_settings_invalid raise SystemExit but should not")
def test_with_registry_and_skopeo(has_skopeo_command_mock): has_skopeo_command_mock.return_value = True exit_if_settings_invalid( get_settings( sys_args=[ "OSP16.2", "/tmp/repoOSP16.2", "--registry", "localhost:5000" ], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "jSbJwfCdIfq12gwHAAtg5JXSBTO3wj0xkG7oW3DlqyM7bXahPRrfZlqmSv3BhmAy", "DCI_CS_URL": "https://distributed-ci.io", }, ))
def test_get_settings_add_env_variables(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo6"], env_variables={ "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "jSbJwfCdIfq12gwHAAtg5JXSBTO3wj0xkG7oW3DlqyM7bXahPRrfZlqmSv3BhmAy", "DCI_CS_URL": "https://distributed-ci.io", }, ) assert settings["env_variables"] == { "DCI_CLIENT_ID": "remoteci/9dd94b70-1707-46c5-a2bb-661e8d5d4212", "DCI_API_SECRET": "jSbJwfCdIfq12gwHAAtg5JXSBTO3wj0xkG7oW3DlqyM7bXahPRrfZlqmSv3BhmAy", "DCI_CS_URL": "https://distributed-ci.io", }
def test_filter_files_list_with_debug(): dci_files_list = { "directories": [], "files": [ { "path": "AppStream/x86_64/debug/tree/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.x86_64.rpm", "size": 45052, }, { "path": "AppStream/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, ], "symlinks": [], } settings = get_settings( sys_args=["RHEL-8", "/tmp", "--variant", "AppStream", "--debug" ])["topics"][0] expected_files_list = { "directories": [], "files": [ { "path": "AppStream/x86_64/debug/tree/Packages", "sha256": "6f48f0d285918e502035da74decf447c6bb29898206406a4ed6a92ece94d276a", "name": "PackageKit-command-not-found-debuginfo-1.1.12-2.el8.x86_64.rpm", "size": 45052, }, { "path": "AppStream/x86_64/os/Packages", "sha256": "8fe293470f677bfc6eb04204c47b5e1a0e5d15431ef7ed9dbb269aaea386ed9f", "name": "PackageKit-command-not-found-1.1.12-2.el8.x86_64.rpm", "size": 28616, }, ], "symlinks": [], } assert filter_files_list(dci_files_list, settings) == expected_files_list
def test_exit_if_settings_invalid_with_bad_variants(): with pytest.raises(SystemExit): exit_if_settings_invalid( get_settings( sys_args=[ "RHEL-8", "/tmp/repo9", "--variant", "AppStream", "--variant", "Server", ], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "", }, ))
def test_get_settings_read_arguments_download_everything(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo3", "--all"], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) _topic_equals( settings["topics"][0], { "download_everything": True, "download_folder": "/tmp/repo3", "name": "RHEL-8", "archs": ["x86_64"], "variants": [], }, )
def test_get_settings_read_arguments_download_everything(): settings = get_settings( sys_args=["RHEL-8", "/tmp/repo3", "--all"], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) assert settings["topics"][0] == { "variants": [], "download_everything": True, "download_folder": "/tmp/repo3", "archs": ["x86_64"], "component_id": None, "dci_key_file": ANY, "components": [], "dci_cert_file": ANY, "name": "RHEL-8", "with_debug": False, "registry": None, }
def test_get_settings_local_repo_with_multiple_topics(): test_dir = os.path.dirname(os.path.abspath(__file__)) settings_file_path = os.path.join(test_dir, "data", "settings_v5.yml") settings = get_settings( sys_args=["--settings", settings_file_path], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) assert settings["topics"][0] == { "variants": [], "download_everything": False, "download_folder": "/tmp/repo6", "archs": ["x86_64"], "component_id": None, "dci_key_file": ANY, "components": [], "dci_cert_file": ANY, "name": "RHEL-7.6", "with_debug": False, "registry": None, } assert settings["topics"][1] == { "variants": [], "download_everything": False, "download_folder": "/tmp/repo6", "archs": ["x86_64"], "component_id": None, "dci_key_file": ANY, "components": [], "dci_cert_file": ANY, "name": "RHEL-8.1", "with_debug": False, "registry": None, }
def test_get_settings_read_arguments(): settings = get_settings( sys_args=[ "RHEL-8", "/tmp/repo2", "--arch", "ppc64le", "--variant", "BaseOS" ], env_variables={ "DCI_CLIENT_ID": "", "DCI_API_SECRET": "", "DCI_CS_URL": "" }, ) _topic_equals( settings["topics"][0], { "download_everything": False, "download_folder": "/tmp/repo2", "name": "RHEL-8", "archs": ["ppc64le"], "variants": [{ "name": "BaseOS", "with_debug": False }], }, )