def _load_specific_flink_module_jars(jars_relative_path):
flink_source_root = _find_flink_source_root()
jars_abs_path = flink_source_root + jars_relative_path
specific_jars = glob.glob(jars_abs_path + '/target/flink*.jar')
specific_jars = [
'file://' + specific_jar for specific_jar in specific_jars
]
add_jars_to_context_class_loader(specific_jars)
def set_string(self, key: str, value: str) -> 'Configuration':
"""
Adds the given key/value pair to the configuration object.
:param key: The key of the key/value pair to be added.
:param value: The value of the key/value pair to be added.
"""
jvm = get_gateway().jvm
jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key(
)
classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key(
)
if key in [jars_key, classpaths_key]:
add_jars_to_context_class_loader(value.split(";"))
self._j_configuration.setString(key, value)
return self
def add_jars(self, *jars_path: str):
"""
Adds a list of jar files that will be uploaded to the cluster and referenced by the job.
:param jars_path: Path of jars.
"""
add_jars_to_context_class_loader(jars_path)
jvm = get_gateway().jvm
jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key()
env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \
.getEnvironmentConfig(self._j_stream_execution_environment)
old_jar_paths = env_config.getString(jars_key, None)
jars_path = jvm.PythonDependencyUtils.FILE_DELIMITER.join(jars_path)
if old_jar_paths is not None:
jars_path = jvm.PythonDependencyUtils.FILE_DELIMITER.join([old_jar_paths, jars_path])
env_config.setString(jars_key, jars_path)
def add_classpaths(self, *classpaths: str):
"""
Adds a list of URLs that are added to the classpath of each user code classloader of the
program. Paths must specify a protocol (e.g. file://) and be accessible on all nodes
:param classpaths: Classpaths that will be added.
"""
add_jars_to_context_class_loader(classpaths)
jvm = get_gateway().jvm
classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key()
env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \
.getEnvironmentConfig(self._j_stream_execution_environment)
old_classpaths = env_config.getString(classpaths_key, None)
classpaths = jvm.PythonDependencyUtils.FILE_DELIMITER.join(classpaths)
if old_classpaths is not None:
classpaths = jvm.PythonDependencyUtils.FILE_DELIMITER.join([old_classpaths, classpaths])
env_config.setString(classpaths_key, classpaths)
