def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ sparkConf = init_spark_conf().setMaster("local[4]").setAppName( "test feature set") self.sc = init_nncontext(sparkConf)
def test_dataframe_shard_size(self): from zoo.orca import OrcaContext OrcaContext._shard_size = 3 sc = init_nncontext() rdd = sc.range(0, 10) from pyspark.sql import SparkSession spark = SparkSession(sc) from pyspark.ml.linalg import DenseVector df = rdd.map(lambda x: (DenseVector(np.random.randn(1, ).astype(np.float)), int(np.random.randint(0, 1, size=())))).toDF( ["feature", "label"]) config = {"lr": 0.8} trainer = Estimator.from_keras(model_creator=model_creator, verbose=True, config=config, workers_per_node=2) trainer.fit(df, epochs=1, batch_size=4, steps_per_epoch=25, feature_cols=["feature"], label_cols=["label"]) trainer.evaluate(df, batch_size=4, num_steps=25, feature_cols=["feature"], label_cols=["label"]) trainer.predict(df, feature_cols=["feature"]).collect()
def test_openvino(self): with tempfile.TemporaryDirectory() as local_path: model_url = data_url + "/analytics-zoo-data/openvino2020_resnet50.tar" model_path = maybe_download("openvino2020_resnet50.tar", local_path, model_url) cmd = "tar -xvf " + model_path + " -C " + local_path subprocess.Popen(cmd.split()) model_path = os.path.join( local_path, "openvino2020_resnet50/resnet_v1_50.xml") est = Estimator.from_openvino(model_path=model_path) # ndarray input_data = np.random.random([20, 4, 3, 224, 224]) result = est.predict(input_data) print(result) # xshards input_data_list = [ np.random.random([1, 4, 3, 224, 224]), np.random.random([2, 4, 3, 224, 224]) ] sc = init_nncontext() rdd = sc.parallelize(input_data_list, numSlices=2) shards = SparkXShards(rdd) def pre_processing(images): return {"x": images} shards = shards.transform_shard(pre_processing) result = est.predict(shards) result_c = result.collect() print(result_c)
def create_sc(self, submit_args, conf): submit_args = submit_args + " pyspark-shell" os.environ["PYSPARK_SUBMIT_ARGS"] = submit_args spark_conf = init_spark_conf(conf) sc = init_nncontext(conf=spark_conf, spark_log_level=self.spark_log_level, redirect_spark_log=self.redirect_spark_log) return sc
def test_estimator_graph_dataframe(self): tf.reset_default_graph() model = SimpleModel() file_path = os.path.join(resource_path, "orca/learn/ncf.csv") sc = init_nncontext() sqlcontext = SQLContext(sc) df = sqlcontext.read.csv(file_path, header=True, inferSchema=True) est = Estimator.from_graph(inputs=[model.user, model.item], labels=[model.label], outputs=[model.logits], loss=model.loss, optimizer=tf.train.AdamOptimizer(), metrics={"loss": model.loss}) est.fit(data=df, batch_size=8, epochs=10, feature_cols=['user', 'item'], labels_cols=['label'], validation_data=df) result = est.evaluate(df, batch_size=4, feature_cols=['user', 'item'], labels_cols=['label']) print(result) prediction_df = est.predict(df, batch_size=4, feature_cols=['user', 'item']) assert 'prediction' in prediction_df.columns predictions = prediction_df.collect() assert len(predictions) == 10
def test_partition_num_less_than_workers(self): sc = init_nncontext() rdd = sc.range(200, numSlices=1) assert rdd.getNumPartitions() == 1 from pyspark.sql import SparkSession spark = SparkSession(sc) from pyspark.ml.linalg import DenseVector df = rdd.map(lambda x: (DenseVector(np.random.randn(1, ).astype(np.float)), int(np.random.randint(0, 1, size=())))).toDF( ["feature", "label"]) config = {"lr": 0.8} trainer = Estimator.from_keras(model_creator=model_creator, verbose=True, config=config, workers_per_node=2) assert df.rdd.getNumPartitions() < trainer.num_workers trainer.fit(df, epochs=1, batch_size=4, steps_per_epoch=25, validation_data=df, validation_steps=1, feature_cols=["feature"], label_cols=["label"]) trainer.evaluate(df, batch_size=4, num_steps=25, feature_cols=["feature"], label_cols=["label"]) trainer.predict(df, feature_cols=["feature"]).collect()
def _create_sc(self, submit_args, conf): os.environ['PYSPARK_SUBMIT_ARGS'] = submit_args zoo_conf = init_spark_conf(conf) sc = init_nncontext(conf=zoo_conf, spark_log_level=self.spark_log_level, redirect_spark_log=self.redirect_spark_log) return sc
def test_num_part_data_diff_val_data(self): sc = init_nncontext() rdd = sc.range(200, numSlices=10) val_rdd = sc.range(60, numSlices=8) from pyspark.sql import SparkSession spark = SparkSession(sc) from pyspark.ml.linalg import DenseVector df = rdd.map(lambda x: (DenseVector(np.random.randn(1, ).astype(np.float)), int(np.random.randint(0, 1, size=())))).toDF( ["feature", "label"]) val_df = val_rdd.map(lambda x: (DenseVector(np.random.randn(1,).astype(np.float)), int(np.random.randint(0, 1, size=()))))\ .toDF(["feature", "label"]) config = {"lr": 0.8} trainer = Estimator.from_keras(model_creator=model_creator, verbose=True, config=config, workers_per_node=2) assert df.rdd.getNumPartitions() > trainer.num_workers assert df.rdd.getNumPartitions() != val_df.rdd.getNumPartitions() trainer.fit(df, epochs=1, batch_size=4, steps_per_epoch=25, validation_data=val_df, validation_steps=1, feature_cols=["feature"], label_cols=["label"])
def main(data_num): sc = init_nncontext() # get data, pre-process and create TFDataset (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test") images_data = (images_data[:data_num] - mnist.TRAIN_MEAN) / mnist.TRAIN_STD labels_data = labels_data[:data_num].astype(np.int32) dataset = TFDataset.from_ndarrays((images_data, labels_data), batch_per_thread=20) # construct the model from TFDataset images, labels = dataset.tensors labels = tf.squeeze(labels) with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(images, num_classes=10, is_training=False) predictions = tf.to_int32(tf.argmax(logits, axis=1)) correct = tf.expand_dims(tf.to_int32(tf.equal(predictions, labels)), axis=1) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver.restore(sess, "/tmp/lenet/model") predictor = TFPredictor(sess, [correct]) accuracy = predictor.predict().mean() print("predict accuracy is %s" % accuracy)
def main(option): sc = init_nncontext() def input_fn(mode, params): if mode == tf.estimator.ModeKeys.TRAIN: image_set = ImageSet.read(params["image_path"], sc=sc, with_label=True, one_based_label=False) train_transformer = ChainedPreprocessing([ ImageBytesToMat(), ImageResize(256, 256), ImageRandomCrop(224, 224), ImageRandomPreprocessing(ImageHFlip(), 0.5), ImageChannelNormalize(0.485, 0.456, 0.406, 0.229, 0.224, 0.225), ImageMatToTensor(to_RGB=True, format="NHWC"), ImageSetToSample(input_keys=["imageTensor"], target_keys=["label"]) ]) feature_set = FeatureSet.image_frame(image_set.to_image_frame()) feature_set = feature_set.transform(train_transformer) feature_set = feature_set.transform(ImageFeatureToSample()) dataset = TFDataset.from_feature_set(feature_set, features=(tf.float32, [224, 224, 3]), labels=(tf.int32, [1]), batch_size=16) else: raise NotImplementedError return dataset def model_fn(features, labels, mode, params): from nets import inception slim = tf.contrib.slim labels = tf.squeeze(labels, axis=1) with slim.arg_scope(inception.inception_v1_arg_scope()): logits, end_points = inception.inception_v1( features, num_classes=int(params["num_classes"]), is_training=True) if mode == tf.estimator.ModeKeys.TRAIN: loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) return TFEstimatorSpec(mode, predictions=logits, loss=loss) else: raise NotImplementedError estimator = TFEstimator(model_fn, tf.train.AdamOptimizer(), params={ "image_path": option.image_path, "num_classes": option.num_classes }) estimator.train(input_fn, steps=100)
def setup_method(self, method): """ setup any state tied to the execution of the given method in a class. setup_method is invoked for every test method of a class. """ sparkConf = init_spark_conf().setMaster("local[1]").setAppName("testEstimator") self.sc = init_nncontext(sparkConf) self.sqlContext = SQLContext(self.sc) assert(self.sc.appName == "testEstimator")
def _create_sc(self, submit_args, conf): from pyspark.sql import SparkSession print("pyspark_submit_args is: {}".format(submit_args)) os.environ['PYSPARK_SUBMIT_ARGS'] = submit_args zoo_conf = init_spark_conf(conf) sc = init_nncontext(conf=zoo_conf, redirect_spark_log=self.redirect_spark_log) sc.setLogLevel(self.spark_log_level) return sc
def main(): sc = init_nncontext() global_batch_size = 256 loss = create_model(creat_dataset(global_batch_size)) optimizer = TFOptimizer.from_loss(loss, SGD(1e-3), model_dir="/tmp/lenet/") optimizer.optimize(end_trigger=MaxIteration(20))
def init_spark_on_local(self, cores, conf=None, python_location=None): print("Start to getOrCreate SparkContext") os.environ['PYSPARK_PYTHON'] = \ python_location if python_location else self._detect_python_location() master = "local[{}]".format(cores) zoo_conf = init_spark_conf(conf).setMaster(master) sc = init_nncontext(conf=zoo_conf, redirect_spark_log=self.redirect_spark_log) sc.setLogLevel(self.spark_log_level) print("Successfully got a SparkContext") return sc
def main(): sc = init_nncontext() def model_fn(features, labels, mode): from nets import lenet slim = tf.contrib.slim with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(features, num_classes=10, is_training=True) if mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN: loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) return TFEstimatorSpec(mode, predictions=logits, loss=loss) else: return TFEstimatorSpec(mode, predictions=logits) def input_fn(mode): if mode == tf.estimator.ModeKeys.TRAIN: training_rdd = get_data_rdd("train", sc) dataset = TFDataset.from_rdd(training_rdd, features=(tf.float32, [28, 28, 1]), labels=(tf.int32, []), batch_size=320) elif mode == tf.estimator.ModeKeys.EVAL: testing_rdd = get_data_rdd("test", sc) dataset = TFDataset.from_rdd(testing_rdd, features=(tf.float32, [28, 28, 1]), labels=(tf.int32, []), batch_size=320) else: testing_rdd = get_data_rdd("test", sc).map(lambda x: x[0]) dataset = TFDataset.from_rdd(testing_rdd, features=(tf.float32, [28, 28, 1]), batch_per_thread=80) return dataset estimator = TFEstimator(model_fn, tf.train.AdamOptimizer(), model_dir="/tmp/estimator") estimator.train(input_fn, steps=60000 // 320) metrics = estimator.evaluate(input_fn, ["acc"]) print(metrics) predictions = estimator.predict(input_fn) print(predictions.first())
def test_openvino_predict_spark_df(self): from pyspark.sql import SparkSession sc = init_nncontext() spark = SparkSession(sc) rdd = sc.range(0, 20, numSlices=2) input_df = rdd.map( lambda x: (np.random.random([1, 4, 3, 224, 224]).tolist())).toDF( ["feature"]) result_df = self.est.predict(input_df, feature_cols=["feature"]) assert np.array(result_df.select("prediction").first()).shape == (1, 4, 1000) assert result_df.count() == 20
def test_dataframe_predict(self): sc = init_nncontext() rdd = sc.parallelize(range(20)) df = rdd.map(lambda x: ([float(x)] * 5, [int(np.random.randint(0, 2, size=()))])).toDF( ["feature", "label"]) estimator = get_estimator(workers_per_node=2, model_fn=lambda config: IdentityNet()) result = estimator.predict(df, batch_size=4, feature_cols=["feature"]) expr = "sum(cast(feature <> to_array(prediction) as int)) as error" assert result.selectExpr(expr).first()["error"] == 0
def main(): sc = init_nncontext() def model_fn(features, labels, mode): from nets import lenet slim = tf.contrib.slim with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(features, num_classes=10, is_training=True) if mode == tf.estimator.ModeKeys.EVAL or mode == tf.estimator.ModeKeys.TRAIN: loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) optimizer = ZooOptimizer(tf.train.AdamOptimizer()) train_op = optimizer.minimize(loss) return tf.estimator.EstimatorSpec(mode, predictions=logits, loss=loss, train_op=train_op) else: return tf.estimator.EstimatorSpec(mode, predictions=logits) def input_fn(mode): if mode == tf.estimator.ModeKeys.TRAIN: training_data = get_data("train") dataset = TFDataset.from_ndarrays(training_data, batch_size=320) elif mode == tf.estimator.ModeKeys.EVAL: testing_data = get_data("test") dataset = TFDataset.from_ndarrays(testing_data, batch_per_thread=80) else: images, _ = get_data("test") dataset = TFDataset.from_ndarrays(images, batch_per_thread=80) return dataset estimator = TFEstimator.from_model_fn(model_fn, model_dir="/tmp/estimator") estimator.train(input_fn, steps=10) metrics = estimator.evaluate(input_fn, ["acc"]) print(metrics) predictions = estimator.predict(input_fn) print(predictions.first()) print("finished...") sc.stop()
def test_estimator_graph_dataframe_exception(self): tf.reset_default_graph() model = SimpleModel() file_path = os.path.join(resource_path, "orca/learn/ncf.csv") sc = init_nncontext() sqlcontext = SQLContext(sc) df = sqlcontext.read.csv(file_path, header=True, inferSchema=True) est = Estimator.from_graph(inputs=[model.user, model.item], labels=[model.label], outputs=[model.logits], loss=model.loss, optimizer=tf.train.AdamOptimizer(), metrics={"loss": model.loss}) with self.assertRaises(Exception) as context: est.fit(data=df, batch_size=8, epochs=10, feature_cols=['user', 'item'], validation_data=df) self.assertTrue( 'label columns is None; it should not be None in training' in str( context.exception)) est.fit(data=df, batch_size=8, epochs=10, feature_cols=['user', 'item'], labels_cols=['label']) with self.assertRaises(Exception) as context: predictions = est.predict(df, batch_size=4).collect() self.assertTrue( 'feature columns is None; it should not be None in prediction' in str(context.exception)) with self.assertRaises(Exception) as context: est.fit(data=df, batch_size=8, epochs=10, feature_cols=['user', 'item'], labels_cols=['label'], validation_data=[1, 2, 3]) self.assertTrue( 'train data and validation data should be both Spark DataFrame' in str(context.exception))
def main(data_num): data = Input(shape=[28, 28, 1]) x = Flatten()(data) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) predictions = Dense(10, activation='softmax')(x) model = Model(inputs=data, outputs=predictions) model.load_weights("/tmp/mnist_keras.h5") if DISTRIBUTED: # using RDD api to do distributed evaluation sc = init_nncontext() # get data, pre-process and create TFDataset (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test") image_rdd = sc.parallelize(images_data[:data_num]) labels_rdd = sc.parallelize(labels_data[:data_num]) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD)]) dataset = TFDataset.from_rdd(rdd, names=["features"], shapes=[[28, 28, 1]], types=[tf.float32], batch_per_thread=20) predictor = TFPredictor.from_keras(model, dataset) accuracy = predictor.predict().zip(labels_rdd).map( lambda x: np.argmax(x[0]) == x[1]).mean() print("predict accuracy is %s" % accuracy) else: # using keras api for local evaluation model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy']) (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", "test") images_data = normalizer(images_data, mnist.TRAIN_MEAN, mnist.TRAIN_STD) result = model.evaluate(images_data, labels_data) print(model.metrics_names) print(result)
def main(max_epoch, data_num): sc = init_nncontext() # get data, pre-process and create TFDataset def get_data_rdd(dataset): (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset) image_rdd = sc.parallelize(images_data[:data_num]) labels_rdd = sc.parallelize(labels_data[:data_num]) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) return rdd training_rdd = get_data_rdd("train") testing_rdd = get_data_rdd("test") dataset = TFDataset.from_rdd(training_rdd, names=["features", "labels"], shapes=[[28, 28, 1], []], types=[tf.float32, tf.int32], batch_size=280, val_rdd=testing_rdd) # construct the model from TFDataset images, labels = dataset.tensors with slim.arg_scope(lenet.lenet_arg_scope()): logits, end_points = lenet.lenet(images, num_classes=10, is_training=True) loss = tf.reduce_mean( tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=labels)) # create a optimizer optimizer = TFOptimizer(loss, Adam(1e-3), val_outputs=[logits], val_labels=[labels], val_method=Top1Accuracy()) optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) # kick off training optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) saver = tf.train.Saver() saver.save(optimizer.sess, "/tmp/lenet/")
def test_openvino_predict_xshards(self): input_data_list = [np.array([self.input] * 4), np.array([self.input] * 2)] sc = init_nncontext() rdd = sc.parallelize(input_data_list, numSlices=2) shards = SparkXShards(rdd) def pre_processing(images): return {"x": images} shards = shards.transform_shard(pre_processing) result = self.est.predict(shards) result_c = result.collect() assert isinstance(result, SparkXShards) assert result_c[0]["prediction"].shape == (4, 1000) assert result_c[1]["prediction"].shape == (2, 1000) assert self.check_result(result_c[0]["prediction"], 4) assert self.check_result(result_c[1]["prediction"], 2)
def test_dataframe_predict(self): sc = init_nncontext() rdd = sc.parallelize(range(100)) from pyspark.sql import SparkSession spark = SparkSession(sc) df = rdd.map(lambda x: ([float(x)] * 50, [int(np.random.randint(0, 2, size=()))])).toDF( ["feature", "label"]) estimator = get_estimator(workers_per_node=2, model_fn=lambda config: IdentityNet()) result = estimator.predict(df, batch_size=4, feature_cols=["feature"]) result = np.concatenate( [shard["prediction"] for shard in result.collect()]) assert np.array_equal(result, np.array(range(100)).astype(np.float))
def write(path, generator, schema, block_size=1000, write_mode="overwrite", **kwargs): """ Take each record in the generator and write it to a parquet file. **generator** Each record in the generator is a dict, the key is a string and will be the column name of saved parquet record and the value is the data. **schema** schema defines the name, dtype, shape of a column, as well as the feature type of a column. The feature type, defines how to encode and decode the column value. There are three kinds of feature type: 1. Scalar, such as a int or float number, or a string, which can be directly mapped to a parquet type 2. NDarray, which takes a np.ndarray and save it serialized bytes. The corresponding parquet type is BYTE_ARRAY . 3. Image, which takes a string representing a image file in local file system and save the raw file content bytes. The corresponding parquet type is BYTE_ARRAY. :param path: the output path, e.g. file:///output/path, hdfs:///output/path :param generator: generate a dict, whose key is a string and value is one of (a scalar value, ndarray, image file path) :param schema: a dict, whose key is a string, value is one of (schema_field.Scalar, schema_field.NDarray, schema_field.Image) :param kwargs: other args """ sc = init_nncontext() spark = SparkSession(sc) node_num, core_num = get_node_and_core_number() for i, chunk in enumerate(chunks(generator, block_size)): chunk_path = os.path.join(path, f"chunk={i}") rows_rdd = sc.parallelize(chunk, core_num * node_num)\ .map(lambda x: dict_to_row(schema, x)) spark.createDataFrame(rows_rdd).write.mode(write_mode).parquet( chunk_path) metadata_path = os.path.join(path, "_orca_metadata") write_text(metadata_path, encode_schema(schema))
def test_xshards_predict(self): sc = init_nncontext() rdd = sc.range(0, 110).map(lambda x: np.array([x] * 50)) shards = rdd.mapPartitions(lambda iter: chunks(iter, 5)).map( lambda x: {"x": np.stack(x)}) shards = SparkXShards(shards) estimator = get_estimator(workers_per_node=2, model_fn=lambda config: IdentityNet()) result_shards = estimator.predict(shards, batch_size=4) result = np.concatenate( [shard["prediction"] for shard in result_shards.collect()]) expected_result = np.concatenate( [shard["x"] for shard in result_shards.collect()]) assert np.array_equal(result, expected_result)
def test_dataframe_train_eval(self): sc = init_nncontext() rdd = sc.range(0, 100) df = rdd.map(lambda x: (np.random.randn(50).astype(np.float).tolist( ), [int(np.random.randint(0, 2, size=()))])).toDF(["feature", "label"]) estimator = get_estimator(workers_per_node=2) estimator.fit(df, batch_size=4, epochs=2, feature_cols=["feature"], label_cols=["label"]) estimator.evaluate(df, batch_size=4, feature_cols=["feature"], label_cols=["label"])
def main(max_epoch): _ = init_nncontext() (training_images_data, training_labels_data) = mnist.read_data_sets("/tmp/mnist", "train") (testing_images_data, testing_labels_data) = mnist.read_data_sets("/tmp/mnist", "test") training_images_data = (training_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD testing_images_data = (testing_images_data - mnist.TRAIN_MEAN) / mnist.TRAIN_STD model = tf.keras.Sequential([ tf.keras.layers.Flatten(input_shape=(28, 28, 1)), tf.keras.layers.Dense(64, activation='relu'), tf.keras.layers.Dense(64, activation='relu'), tf.keras.layers.Dense(10, activation='softmax'), ]) model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy']) keras_model = KerasModel(model) keras_model.fit(training_images_data, training_labels_data, validation_data=(testing_images_data, testing_labels_data), epochs=max_epoch, batch_size=320, distributed=True) result = keras_model.evaluate(testing_images_data, testing_labels_data, distributed=True, batch_per_thread=80) print(result) # >> [0.08865142822265625, 0.9722] # the following assert is used for internal testing assert result['acc Top1Accuracy'] > 0.95 keras_model.save_weights("/tmp/mnist_keras.h5")
def main(max_epoch, data_num): sc = init_nncontext() # get data, pre-process and create TFDataset def get_data_rdd(dataset): (images_data, labels_data) = mnist.read_data_sets("/tmp/mnist", dataset) image_rdd = sc.parallelize(images_data[:data_num]) labels_rdd = sc.parallelize(labels_data[:data_num]) rdd = image_rdd.zip(labels_rdd) \ .map(lambda rec_tuple: [normalizer(rec_tuple[0], mnist.TRAIN_MEAN, mnist.TRAIN_STD), np.array(rec_tuple[1])]) return rdd training_rdd = get_data_rdd("train") testing_rdd = get_data_rdd("test") dataset = TFDataset.from_rdd(training_rdd, names=["features", "labels"], shapes=[[28, 28, 1], []], types=[tf.float32, tf.int32], batch_size=280, val_rdd=testing_rdd ) data = Input(shape=[28, 28, 1]) x = Flatten()(data) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) predictions = Dense(10, activation='softmax')(x) model = Model(input=data, output=predictions) model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy']) optimizer = TFOptimizer.from_keras(model, dataset) optimizer.set_train_summary(TrainSummary("/tmp/az_lenet", "lenet")) optimizer.set_val_summary(ValidationSummary("/tmp/az_lenet", "lenet")) # kick off training optimizer.optimize(end_trigger=MaxEpoch(max_epoch)) saver = tf.train.Saver() saver.save(optimizer.sess, "/tmp/lenet/")
def test_openvino_predict_xshards(self): input_data_list = [ np.random.random([1, 4, 3, 224, 224]), np.random.random([2, 4, 3, 224, 224]) ] sc = init_nncontext() rdd = sc.parallelize(input_data_list, numSlices=2) shards = SparkXShards(rdd) def pre_processing(images): return {"x": images} shards = shards.transform_shard(pre_processing) result = self.est.predict(shards) result_c = result.collect() assert isinstance(result, SparkXShards) assert result_c[0]["prediction"].shape == (1, 4, 1000) assert result_c[1]["prediction"].shape == (2, 4, 1000)
def test_spark_xshards(self): from zoo import init_nncontext from zoo.orca.data import SparkXShards estimator = get_estimator(workers_per_node=1) sc = init_nncontext() x_rdd = sc.parallelize(np.random.rand(4000, 1, 50).astype(np.float32)) # torch 1.7.1+ requires target size same as output size, which is (batch, 1) y_rdd = sc.parallelize( np.random.randint(0, 2, size=(4000, 1, 1)).astype(np.float32)) rdd = x_rdd.zip(y_rdd).map(lambda x_y: {'x': x_y[0], 'y': x_y[1]}) train_rdd, val_rdd = rdd.randomSplit([0.9, 0.1]) train_xshards = SparkXShards(train_rdd) val_xshards = SparkXShards(val_rdd) train_stats = estimator.fit(train_xshards, batch_size=256, epochs=2) print(train_stats) val_stats = estimator.evaluate(val_xshards, batch_size=128) print(val_stats) estimator.shutdown()