def _load_specific_flink_module_jars(jars_relative_path):
flink_source_root = _find_flink_source_root()
jars_abs_path = flink_source_root + jars_relative_path
specific_jars = glob.glob(jars_abs_path + '/target/flink*.jar')
specific_jars = [
'file://' + specific_jar for specific_jar in specific_jars
]
add_jars_to_context_class_loader(specific_jars)
def set(self, key: str, value: str) -> 'TableConfig':
"""
Sets a string-based value for the given string-based key.
The value will be parsed by the framework on access.
"""
self._j_table_config.set(key, value)
jvm = get_gateway().jvm
jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key()
classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key()
if key in [jars_key, classpaths_key]:
add_jars_to_context_class_loader(value.split(";"))
return self
def add_jars(self, *jars_path: str):
"""
Adds a list of jar files that will be uploaded to the cluster and referenced by the job.
:param jars_path: Path of jars.
"""
add_jars_to_context_class_loader(jars_path)
jvm = get_gateway().jvm
jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key()
env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \
.getEnvironmentConfig(self._j_stream_execution_environment)
old_jar_paths = env_config.getString(jars_key, None)
joined_jars_path = ';'.join(jars_path)
if old_jar_paths and old_jar_paths.strip():
joined_jars_path = ';'.join([old_jar_paths, joined_jars_path])
env_config.setString(jars_key, joined_jars_path)
def set_string(self, key: str, value: str) -> 'Configuration':
"""
Adds the given key/value pair to the configuration object.
:param key: The key of the key/value pair to be added.
:param value: The value of the key/value pair to be added.
"""
jvm = get_gateway().jvm
jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key(
)
classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key(
)
if key in [jars_key, classpaths_key]:
add_jars_to_context_class_loader(value.split(";"))
self._j_configuration.setString(key, value)
return self
def add_classpaths(self, *classpaths: str):
"""
Adds a list of URLs that are added to the classpath of each user code classloader of the
program. Paths must specify a protocol (e.g. file://) and be accessible on all nodes
:param classpaths: Classpaths that will be added.
"""
add_jars_to_context_class_loader(classpaths)
jvm = get_gateway().jvm
classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key()
env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \
.getEnvironmentConfig(self._j_stream_execution_environment)
old_classpaths = env_config.getString(classpaths_key, None)
joined_classpaths = ';'.join(list(classpaths))
if old_classpaths and old_classpaths.strip():
joined_classpaths = ';'.join([old_classpaths, joined_classpaths])
env_config.setString(classpaths_key, joined_classpaths)
"target")
paths = glob.glob(
os.path.join(target_dir,
"flink-ml-tensorflow-*-jar-with-dependencies.jar"))
if len(paths) < 1:
raise RuntimeError(
"Cannot find flink-ml-tensorflow jar, please make sure you have run `mvn package`"
)
elif len(paths) >= 2:
raise RuntimeError(
"Found more than one flink-ml-tensorflow jar {}".format(paths))
# logger.info("Found flink-ml-tensorflow jar at {}".format(paths[0]))
return paths[0]
add_jars_to_context_class_loader(["file://{}".format(find_jar_path())])
import logging
import shutil
import time
import unittest
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.table import StreamTableEnvironment, DataTypes, Table, TableDescriptor, Schema
from flink_ml_tensorflow.tensorflow_TFConfig import TFConfig
from flink_ml_tensorflow.tensorflow_on_flink_ml import TensorflowEstimator, TensorflowModel
from flink_ml_tensorflow.tensorflow_on_flink_mlconf import MLCONSTANTS
logger = logging.getLogger(__name__)
