def _load_specific_flink_module_jars(jars_relative_path): flink_source_root = _find_flink_source_root() jars_abs_path = flink_source_root + jars_relative_path specific_jars = glob.glob(jars_abs_path + '/target/flink*.jar') specific_jars = [ 'file://' + specific_jar for specific_jar in specific_jars ] add_jars_to_context_class_loader(specific_jars)
def set(self, key: str, value: str) -> 'TableConfig': """ Sets a string-based value for the given string-based key. The value will be parsed by the framework on access. """ self._j_table_config.set(key, value) jvm = get_gateway().jvm jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key() classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key() if key in [jars_key, classpaths_key]: add_jars_to_context_class_loader(value.split(";")) return self
def add_jars(self, *jars_path: str): """ Adds a list of jar files that will be uploaded to the cluster and referenced by the job. :param jars_path: Path of jars. """ add_jars_to_context_class_loader(jars_path) jvm = get_gateway().jvm jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key() env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \ .getEnvironmentConfig(self._j_stream_execution_environment) old_jar_paths = env_config.getString(jars_key, None) joined_jars_path = ';'.join(jars_path) if old_jar_paths and old_jar_paths.strip(): joined_jars_path = ';'.join([old_jar_paths, joined_jars_path]) env_config.setString(jars_key, joined_jars_path)
def set_string(self, key: str, value: str) -> 'Configuration': """ Adds the given key/value pair to the configuration object. :param key: The key of the key/value pair to be added. :param value: The value of the key/value pair to be added. """ jvm = get_gateway().jvm jars_key = jvm.org.apache.flink.configuration.PipelineOptions.JARS.key( ) classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key( ) if key in [jars_key, classpaths_key]: add_jars_to_context_class_loader(value.split(";")) self._j_configuration.setString(key, value) return self
def add_classpaths(self, *classpaths: str): """ Adds a list of URLs that are added to the classpath of each user code classloader of the program. Paths must specify a protocol (e.g. file://) and be accessible on all nodes :param classpaths: Classpaths that will be added. """ add_jars_to_context_class_loader(classpaths) jvm = get_gateway().jvm classpaths_key = jvm.org.apache.flink.configuration.PipelineOptions.CLASSPATHS.key() env_config = jvm.org.apache.flink.python.util.PythonConfigUtil \ .getEnvironmentConfig(self._j_stream_execution_environment) old_classpaths = env_config.getString(classpaths_key, None) joined_classpaths = ';'.join(list(classpaths)) if old_classpaths and old_classpaths.strip(): joined_classpaths = ';'.join([old_classpaths, joined_classpaths]) env_config.setString(classpaths_key, joined_classpaths)
"target") paths = glob.glob( os.path.join(target_dir, "flink-ml-tensorflow-*-jar-with-dependencies.jar")) if len(paths) < 1: raise RuntimeError( "Cannot find flink-ml-tensorflow jar, please make sure you have run `mvn package`" ) elif len(paths) >= 2: raise RuntimeError( "Found more than one flink-ml-tensorflow jar {}".format(paths)) # logger.info("Found flink-ml-tensorflow jar at {}".format(paths[0])) return paths[0] add_jars_to_context_class_loader(["file://{}".format(find_jar_path())]) import logging import shutil import time import unittest from pyflink.datastream import StreamExecutionEnvironment from pyflink.table import StreamTableEnvironment, DataTypes, Table, TableDescriptor, Schema from flink_ml_tensorflow.tensorflow_TFConfig import TFConfig from flink_ml_tensorflow.tensorflow_on_flink_ml import TensorflowEstimator, TensorflowModel from flink_ml_tensorflow.tensorflow_on_flink_mlconf import MLCONSTANTS logger = logging.getLogger(__name__)