def test_init(self): conf = Configuration() self.assertEqual(conf.to_dict(), dict()) conf.set_string("k1", "v1") conf2 = Configuration(conf) self.assertEqual(conf2.to_dict(), {"k1": "v1"})
def test_to_dict(self): conf = Configuration() conf.set_string("k1", "v1") conf.set_integer("k2", 1) conf.set_float("k3", 1.2) conf.set_boolean("k4", True) target_dict = conf.to_dict() self.assertEqual(target_dict, { "k1": "v1", "k2": "1", "k3": "1.2", "k4": "true" })
class DependencyManagerTests(PyFlinkTestCase): def setUp(self): self.j_env = TestEnv() self.config = Configuration() self.dependency_manager = DependencyManager(self.config, self.j_env) def test_add_python_file(self): self.dependency_manager.add_python_file("tmp_dir/test_file1.py") self.dependency_manager.add_python_file("tmp_dir/test_file2.py") self.dependency_manager.add_python_file("tmp_dir/test_dir") self.assertEqual( { "python_file_0_{uuid}": "test_file1.py", "python_file_1_{uuid}": "test_file2.py", "python_file_2_{uuid}": "test_dir" }, json.loads( replace_uuid( self.config.to_dict()[DependencyManager.PYTHON_FILES]))) self.assertEqual( { "python_file_0_{uuid}": "tmp_dir/test_file1.py", "python_file_1_{uuid}": "tmp_dir/test_file2.py", "python_file_2_{uuid}": "tmp_dir/test_dir" }, replace_uuid(self.j_env.to_dict())) def test_set_python_requirements(self): self.dependency_manager.set_python_requirements( "tmp_dir/requirements.txt", "tmp_dir/cache_dir") self.assertEqual( "python_requirements_file_0_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_FILE])) self.assertEqual( "python_requirements_cache_1_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_CACHE])) self.assertEqual( { "python_requirements_file_0_{uuid}": "tmp_dir/requirements.txt", "python_requirements_cache_1_{uuid}": "tmp_dir/cache_dir" }, replace_uuid(self.j_env.to_dict())) # test single parameter and remove old requirements_dir setting self.dependency_manager.set_python_requirements( "tmp_dir/requirements.txt") self.assertEqual( "python_requirements_file_2_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_FILE])) self.assertNotIn(DependencyManager.PYTHON_REQUIREMENTS_CACHE, self.config.to_dict()) self.assertEqual( {"python_requirements_file_2_{uuid}": "tmp_dir/requirements.txt"}, replace_uuid(self.j_env.to_dict())) def test_add_python_archive(self): self.dependency_manager.add_python_archive("tmp_dir/py27.zip") self.dependency_manager.add_python_archive("tmp_dir/venv2.zip", "py37") self.assertEqual( { "python_archive_0_{uuid}": "py27.zip", "python_archive_1_{uuid}": "py37" }, json.loads( replace_uuid( self.config.to_dict()[DependencyManager.PYTHON_ARCHIVES]))) self.assertEqual( { "python_archive_0_{uuid}": "tmp_dir/py27.zip", "python_archive_1_{uuid}": "tmp_dir/venv2.zip" }, replace_uuid(self.j_env.to_dict())) def test_set_get_python_executable(self): table_config = TableConfig() table_config.set_python_executable("/usr/bin/python3") self.assertEqual("/usr/bin/python3", table_config.get_python_executable()) def test_constant_consistency(self): JPythonConfig = get_gateway().jvm.org.apache.flink.python.PythonConfig self.assertEqual(DependencyManager.PYTHON_REQUIREMENTS_CACHE, JPythonConfig.PYTHON_REQUIREMENTS_CACHE) self.assertEqual(DependencyManager.PYTHON_REQUIREMENTS_FILE, JPythonConfig.PYTHON_REQUIREMENTS_FILE) self.assertEqual(DependencyManager.PYTHON_ARCHIVES, JPythonConfig.PYTHON_ARCHIVES) self.assertEqual(DependencyManager.PYTHON_FILES, JPythonConfig.PYTHON_FILES) self.assertEqual(DependencyManager.PYTHON_EXEC, JPythonConfig.PYTHON_EXEC) JPythonDriverEnvUtils = \ get_gateway().jvm.org.apache.flink.client.python.PythonDriverEnvUtils self.assertEqual(DependencyManager.PYFLINK_PY_REQUIREMENTS, JPythonDriverEnvUtils.PYFLINK_PY_REQUIREMENTS) self.assertEqual(DependencyManager.PYFLINK_PY_ARCHIVES, JPythonDriverEnvUtils.PYFLINK_PY_ARCHIVES) self.assertEqual(DependencyManager.PYFLINK_PY_FILES, JPythonDriverEnvUtils.PYFLINK_PY_FILES) self.assertEqual(DependencyManager.PYFLINK_PY_EXECUTABLE, JPythonDriverEnvUtils.PYFLINK_PY_EXECUTABLE) def test_load_from_env(self): dm = DependencyManager system_env = dict() system_env[ dm.PYFLINK_PY_FILES] = "/file1.py\nhdfs://file2.zip\nfile3.egg" system_env[dm.PYFLINK_PY_REQUIREMENTS] = "a.txt\nb_dir" system_env[dm.PYFLINK_PY_EXECUTABLE] = "/usr/local/bin/python" system_env[ dm. PYFLINK_PY_ARCHIVES] = "/py3.zip\nvenv\n/py3.zip\n\ndata.zip\ndata" self.dependency_manager.load_from_env(system_env) configs = self.config.to_dict() python_files = replace_uuid(json.loads(configs[dm.PYTHON_FILES])) python_requirements_file = replace_uuid( configs[dm.PYTHON_REQUIREMENTS_FILE]) python_requirements_cache = replace_uuid( configs[dm.PYTHON_REQUIREMENTS_CACHE]) python_archives = replace_uuid(json.loads(configs[dm.PYTHON_ARCHIVES])) python_exec = configs[dm.PYTHON_EXEC] registered_files = replace_uuid(self.j_env.to_dict()) self.assertEqual( { "python_file_0_{uuid}": "file1.py", "python_file_1_{uuid}": "file2.zip", "python_file_2_{uuid}": "file3.egg" }, python_files) self.assertEqual( { "python_archive_3_{uuid}": "venv", "python_archive_4_{uuid}": "py3.zip", "python_archive_5_{uuid}": "data" }, python_archives) self.assertEqual("python_requirements_file_6_{uuid}", python_requirements_file) self.assertEqual("python_requirements_cache_7_{uuid}", python_requirements_cache) self.assertEqual("/usr/local/bin/python", python_exec) self.assertEqual( { "python_file_0_{uuid}": "/file1.py", "python_file_1_{uuid}": "hdfs://file2.zip", "python_file_2_{uuid}": "file3.egg", "python_archive_3_{uuid}": "/py3.zip", "python_archive_4_{uuid}": "/py3.zip", "python_archive_5_{uuid}": "data.zip", "python_requirements_file_6_{uuid}": "a.txt", "python_requirements_cache_7_{uuid}": "b_dir" }, registered_files)
def create_java_properties(config: Configuration): jvm = get_gateway().jvm properties = jvm.java.util.Properties() for k, v in config.to_dict().items(): properties.put(k, v) return properties
def create_hadoop_configuration(config: Configuration): jvm = get_gateway().jvm hadoop_config = jvm.org.apache.hadoop.conf.Configuration() for k, v in config.to_dict().items(): hadoop_config.set(k, v) return hadoop_config
class DependencyManagerTests(PyFlinkTestCase): def setUp(self): self.j_env = MockedJavaEnv() self.config = Configuration() self.dependency_manager = DependencyManager(self.config, self.j_env) def test_add_python_file(self): self.dependency_manager.add_python_file("tmp_dir/test_file1.py") self.dependency_manager.add_python_file("tmp_dir/test_file2.py") self.dependency_manager.add_python_file("tmp_dir/test_dir") self.assertEqual( { "python_file_0_{uuid}": "test_file1.py", "python_file_1_{uuid}": "test_file2.py", "python_file_2_{uuid}": "test_dir" }, json.loads( replace_uuid( self.config.to_dict()[DependencyManager.PYTHON_FILES]))) self.assertEqual( { "python_file_0_{uuid}": "tmp_dir/test_file1.py", "python_file_1_{uuid}": "tmp_dir/test_file2.py", "python_file_2_{uuid}": "tmp_dir/test_dir" }, replace_uuid(self.j_env.to_dict())) def test_set_python_requirements(self): self.dependency_manager.set_python_requirements( "tmp_dir/requirements.txt", "tmp_dir/cache_dir") self.assertEqual( "python_requirements_file_0_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_FILE])) self.assertEqual( "python_requirements_cache_1_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_CACHE])) self.assertEqual( { "python_requirements_file_0_{uuid}": "tmp_dir/requirements.txt", "python_requirements_cache_1_{uuid}": "tmp_dir/cache_dir" }, replace_uuid(self.j_env.to_dict())) # test single parameter and remove old requirements_dir setting self.dependency_manager.set_python_requirements( "tmp_dir/requirements.txt") self.assertEqual( "python_requirements_file_2_{uuid}", replace_uuid(self.config.to_dict()[ DependencyManager.PYTHON_REQUIREMENTS_FILE])) self.assertNotIn(DependencyManager.PYTHON_REQUIREMENTS_CACHE, self.config.to_dict()) self.assertEqual( {"python_requirements_file_2_{uuid}": "tmp_dir/requirements.txt"}, replace_uuid(self.j_env.to_dict())) def test_add_python_archive(self): self.dependency_manager.add_python_archive("tmp_dir/py27.zip") self.dependency_manager.add_python_archive("tmp_dir/venv2.zip", "py37") self.assertEqual( { "python_archive_0_{uuid}": "py27.zip", "python_archive_1_{uuid}": "py37" }, json.loads( replace_uuid( self.config.to_dict()[DependencyManager.PYTHON_ARCHIVES]))) self.assertEqual( { "python_archive_0_{uuid}": "tmp_dir/py27.zip", "python_archive_1_{uuid}": "tmp_dir/venv2.zip" }, replace_uuid(self.j_env.to_dict())) def test_set_get_python_executable(self): table_config = TableConfig() table_config.set_python_executable("/usr/bin/python3") self.assertEqual("/usr/bin/python3", table_config.get_python_executable()) def test_constant_consistency(self): JDependencyInfo = get_gateway( ).jvm.org.apache.flink.python.env.PythonDependencyInfo self.assertEqual(DependencyManager.PYTHON_REQUIREMENTS_CACHE, JDependencyInfo.PYTHON_REQUIREMENTS_CACHE) self.assertEqual(DependencyManager.PYTHON_REQUIREMENTS_FILE, JDependencyInfo.PYTHON_REQUIREMENTS_FILE) self.assertEqual(DependencyManager.PYTHON_ARCHIVES, JDependencyInfo.PYTHON_ARCHIVES) self.assertEqual(DependencyManager.PYTHON_FILES, JDependencyInfo.PYTHON_FILES) self.assertEqual(DependencyManager.PYTHON_EXEC, JDependencyInfo.PYTHON_EXEC)