def _load_specific_flink_module_jars(jars_relative_path): flink_source_root = _find_flink_source_root() jars_abs_path = flink_source_root + jars_relative_path specific_jars = glob.glob(jars_abs_path + '/target/flink*.jar') specific_jars = [ 'file://' + specific_jar for specific_jar in specific_jars ] add_jars_to_context_class_loader(specific_jars)
def set_string(self, key: str, value: str) -> 'Configuration': """ Adds the given key/value pair to the configuration object. :param key: The key of the key/value pair to be added. :param value: The value of the key/value pair to be added. """ jvm = get_gateway().jvm jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key( ) classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key( ) if key in [jars_key, classpaths_key]: add_jars_to_context_class_loader(value.split(";")) self._j_configuration.setString(key, value) return self
def add_jars(self, *jars_path: str): """ Adds a list of jar files that will be uploaded to the cluster and referenced by the job. :param jars_path: Path of jars. """ add_jars_to_context_class_loader(jars_path) jvm = get_gateway().jvm jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key() env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \ .getEnvironmentConfig(self._j_stream_execution_environment) old_jar_paths = env_config.getString(jars_key, None) jars_path = jvm.PythonDependencyUtils.FILE_DELIMITER.join(jars_path) if old_jar_paths is not None: jars_path = jvm.PythonDependencyUtils.FILE_DELIMITER.join([old_jar_paths, jars_path]) env_config.setString(jars_key, jars_path)
def add_classpaths(self, *classpaths: str): """ Adds a list of URLs that are added to the classpath of each user code classloader of the program. Paths must specify a protocol (e.g. file://) and be accessible on all nodes :param classpaths: Classpaths that will be added. """ add_jars_to_context_class_loader(classpaths) jvm = get_gateway().jvm classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key() env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \ .getEnvironmentConfig(self._j_stream_execution_environment) old_classpaths = env_config.getString(classpaths_key, None) classpaths = jvm.PythonDependencyUtils.FILE_DELIMITER.join(classpaths) if old_classpaths is not None: classpaths = jvm.PythonDependencyUtils.FILE_DELIMITER.join([old_classpaths, classpaths]) env_config.setString(classpaths_key, classpaths)