def test_estimator_keras_tensorboard(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
temp = tempfile.mkdtemp()
model_dir = os.path.join(temp, "test_model")
est = Estimator.from_keras(keras_model=model, model_dir=model_dir)
assert est.get_train_summary("Loss") is None
assert est.get_validation_summary("Top1Accuracy") is None
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
train_loss = est.get_train_summary("Loss")
assert len(train_loss) > 0
val_scores = est.get_validation_summary("Top1Accuracy")
assert len(val_scores) > 0
tf.reset_default_graph()
# no model dir
model = self.create_model()
est = Estimator.from_keras(keras_model=model)
log_dir = os.path.join(temp, "log")
est.set_tensorboard(log_dir, "test")
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
assert os.path.exists(os.path.join(log_dir, "test/train"))
assert os.path.exists(os.path.join(log_dir, "test/validation"))
train_loss = est.get_train_summary("Loss")
val_scores = est.get_validation_summary("Loss")
assert len(train_loss) > 0
assert len(val_scores) > 0
shutil.rmtree(temp)
def test_estimator_keras_save_load(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_keras(keras_model=model)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
eval_result = est.evaluate(data_shard)
print(eval_result)
temp = tempfile.mkdtemp()
model_path = os.path.join(temp, 'test.h5')
est.save_keras_model(model_path)
tf.reset_default_graph()
from tensorflow.python.keras import models
from zoo.common.utils import load_from_file
def load_func(file_path):
return models.load_model(file_path)
model = load_from_file(load_func, model_path)
est = Estimator.from_keras(keras_model=model)
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
}
return result
data_shard = data_shard.transform_shard(transform)
predictions = est.predict(data_shard).collect()
assert predictions[0]['prediction'].shape[1] == 2
shutil.rmtree(temp)
def test_estimator_keras_xshards_options(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y": df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_keras(keras_model=model)
# train with no validation
est.fit(data=data_shard,
batch_size=8,
epochs=10)
# train with different optimizer
est = Estimator.from_keras(keras_model=model)
est.fit(data=data_shard,
batch_size=8,
epochs=10
)
# train with session config
tf_session_config = tf.ConfigProto(inter_op_parallelism_threads=1,
intra_op_parallelism_threads=1)
est = Estimator.from_keras(keras_model=model)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
session_config=tf_session_config
)
# train with model dir
temp = tempfile.mkdtemp()
model_dir = os.path.join(temp, "model")
est = Estimator.from_keras(keras_model=model, model_dir=model_dir)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
assert len(os.listdir(model_dir)) > 0
shutil.rmtree(temp)
def test_estimator_keras_xshards_checkpoint(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
temp = tempfile.mkdtemp()
model_dir = os.path.join(temp, "test_model")
est = Estimator.from_keras(keras_model=model, model_dir=model_dir)
est.fit(data=data_shard,
batch_size=8,
epochs=6,
validation_data=data_shard,
checkpoint_trigger=SeveralIteration(4))
eval_result = est.evaluate(data_shard)
print(eval_result)
tf.reset_default_graph()
model = self.create_model()
est = Estimator.from_keras(keras_model=model, model_dir=model_dir)
est.load_orca_checkpoint(model_dir)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard,
checkpoint_trigger=SeveralIteration(4))
eval_result = est.evaluate(data_shard)
print(eval_result)
shutil.rmtree(temp)
def test_estimator_keras_xshards_clip(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model_with_clip()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_keras(keras_model=model)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
def test_estimator_keras_xshards_with_mem_type(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_keras(keras_model=model)
OrcaContext.train_data_store = "DISK_2"
est.fit(data=data_shard,
batch_size=4,
epochs=10,
validation_data=data_shard)
eval_result = est.evaluate(data_shard)
print(eval_result)
OrcaContext.train_data_store = "DRAM"
def test_estimator_keras_dataframe_mem_type(self):
tf.reset_default_graph()
model = self.create_model()
sc = init_nncontext()
sqlcontext = SQLContext(sc)
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
df = sqlcontext.read.csv(file_path, header=True, inferSchema=True)
from pyspark.sql.functions import array
df = df.withColumn('user', array('user')) \
.withColumn('item', array('item'))
est = Estimator.from_keras(keras_model=model)
OrcaContext.train_data_store = "DISK_2"
est.fit(data=df,
batch_size=4,
epochs=4,
feature_cols=['user', 'item'],
label_cols=['label'],
validation_data=df)
eval_result = est.evaluate(df,
feature_cols=['user', 'item'],
label_cols=['label'])
assert 'acc Top1Accuracy' in eval_result
prediction_df = est.predict(df,
batch_size=4,
feature_cols=['user', 'item'])
assert 'prediction' in prediction_df.columns
predictions = prediction_df.collect()
assert len(predictions) == 48
OrcaContext.train_data_store = "DRAM"
def main(max_epoch):
sc = init_orca_context(cores=4, memory="2g")
# get DataSet
# as_supervised returns tuple (img, label) instead of dict {'image': img, 'label':label}
mnist_train = tfds.load(name="mnist", split="train", as_supervised=True)
mnist_test = tfds.load(name="mnist", split="test", as_supervised=True)
# Normalizes images, unit8 -> float32
def normalize_img(image, label):
return tf.cast(image, tf.float32) / 255., label
mnist_train = mnist_train.map(
normalize_img, num_parallel_calls=tf.data.experimental.AUTOTUNE)
mnist_test = mnist_test.map(
normalize_img, num_parallel_calls=tf.data.experimental.AUTOTUNE)
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(20,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
input_shape=(28, 28, 1),
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Conv2D(50,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(500, activation='tanh'),
tf.keras.layers.Dense(10, activation='softmax'),
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
est = Estimator.from_keras(keras_model=model)
est.fit(data=mnist_train,
batch_size=320,
epochs=max_epoch,
validation_data=mnist_test)
result = est.evaluate(mnist_test)
print(result)
est.save_keras_model("/tmp/mnist_keras.h5")
stop_orca_context()
def main(max_epoch):
# get DataSet
(train_feature,
train_label), (val_feature,
val_label) = tf.keras.datasets.mnist.load_data()
# tf.data.Dataset.from_tensor_slices is for demo only. For production use, please use
# file-based approach (e.g. tfrecord).
train_dataset = tf.data.Dataset.from_tensor_slices(
(train_feature, train_label))
train_dataset = train_dataset.map(preprocess)
val_dataset = tf.data.Dataset.from_tensor_slices((val_feature, val_label))
val_dataset = val_dataset.map(preprocess)
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(20,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
input_shape=(28, 28, 1),
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Conv2D(50,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(500, activation='tanh'),
tf.keras.layers.Dense(10, activation='softmax'),
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
est = Estimator.from_keras(keras_model=model)
est.fit(data=train_dataset,
batch_size=320,
epochs=max_epoch,
validation_data=val_dataset)
result = est.evaluate(val_dataset)
print(result)
est.save_keras_model("/tmp/mnist_keras.h5")
def test_train_simple(orca_context_fixture):
sc = orca_context_fixture
temp_dir = tempfile.mkdtemp()
try:
_write_ndarrays(images=np.random.randn(500, 28, 28,
1).astype(np.float32),
labels=np.random.randint(0, 10,
(500, )).astype(np.int32),
output_path=temp_dir)
dataset = ParquetDataset.read_as_tf(temp_dir)
def preprocess(data):
return data['image'], data["label"]
dataset = dataset.map(preprocess)
import tensorflow as tf
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(20,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
input_shape=(28, 28, 1),
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Conv2D(50,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(500, activation='tanh'),
tf.keras.layers.Dense(10, activation='softmax'),
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
est = Estimator.from_keras(keras_model=model)
est.fit(data=dataset, batch_size=100, epochs=1)
finally:
shutil.rmtree(temp_dir)
def test_estimator_keras_tf_dataset(self):
tf.reset_default_graph()
model = self.create_model()
dataset = tf.data.Dataset.from_tensor_slices(
(np.random.randint(0, 200, size=(100, 1)),
np.random.randint(0, 50, size=(100, 1)),
np.ones(shape=(100, ), dtype=np.int32)))
dataset = dataset.map(lambda user, item, label: [(user, item), label])
est = Estimator.from_keras(keras_model=model)
est.fit(data=dataset, batch_size=8, epochs=10, validation_data=dataset)
eval_result = est.evaluate(dataset)
assert 'acc Top1Accuracy' in eval_result
def main(max_epoch, dataset_dir):
mnist_train = tfds.load(name="mnist", split="train", data_dir=dataset_dir)
mnist_test = tfds.load(name="mnist", split="test", data_dir=dataset_dir)
mnist_train = mnist_train.map(preprocess)
mnist_test = mnist_test.map(preprocess)
model = tf.keras.Sequential([
tf.keras.layers.Conv2D(20,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
input_shape=(28, 28, 1),
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Conv2D(50,
kernel_size=(5, 5),
strides=(1, 1),
activation='tanh',
padding='valid'),
tf.keras.layers.MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding='valid'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(500, activation='tanh'),
tf.keras.layers.Dense(10, activation='softmax'),
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
est = Estimator.from_keras(keras_model=model)
est.fit(data=mnist_train,
batch_size=320,
epochs=max_epoch,
validation_data=mnist_test,
auto_shard_files=False)
result = est.evaluate(mnist_test, auto_shard_files=False)
print(result)
est.save_keras_model("/tmp/mnist_keras.h5")
def test_estimator_keras_get_model(self):
tf.reset_default_graph()
model = self.create_model()
sc = init_nncontext()
sqlcontext = SQLContext(sc)
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
df = sqlcontext.read.csv(file_path, header=True, inferSchema=True)
from pyspark.sql.functions import array
df = df.withColumn('user', array('user')) \
.withColumn('item', array('item'))
est = Estimator.from_keras(keras_model=model)
est.fit(data=df,
batch_size=4,
epochs=4,
feature_cols=['user', 'item'],
label_cols=['label'],
validation_data=df)
assert est.get_model() is model
def test_submodel_in_keras_squential(self):
mnet = tf.keras.applications.MobileNetV2(input_shape=(160, 160, 3),
include_top=False,
weights='imagenet')
model = tf.keras.Sequential([
mnet,
tf.keras.layers.GlobalAveragePooling2D(),
tf.keras.layers.Dense(1, activation='sigmoid')
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001),
loss='binary_crossentropy',
metrics=['accuracy'])
dataset = tf.data.Dataset.from_tensor_slices(
(np.random.randn(16, 160, 160,
3), np.random.randint(0, 1000, (16, 1))))
est = Estimator.from_keras(keras_model=model)
est.fit(data=dataset, batch_size=4, epochs=1, validation_data=dataset)
def test_estimator_keras_xshards(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
est = Estimator.from_keras(keras_model=model)
est.fit(data=data_shard,
batch_size=8,
epochs=10,
validation_data=data_shard)
eval_result = est.evaluate(data_shard)
print(eval_result)
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
}
return result
data_shard = data_shard.transform_shard(transform)
predictions = est.predict(data_shard).collect()
assert predictions[0]['prediction'].shape[1] == 2
def test_estimator_keras_xshards_disk_featureset_trigger(self):
import zoo.orca.data.pandas
tf.reset_default_graph()
model = self.create_model()
file_path = os.path.join(self.resource_path, "orca/learn/ncf.csv")
data_shard = zoo.orca.data.pandas.read_csv(file_path)
def transform(df):
result = {
"x": (df['user'].to_numpy().reshape([-1, 1]),
df['item'].to_numpy().reshape([-1, 1])),
"y":
df['label'].to_numpy()
}
return result
data_shard = data_shard.transform_shard(transform)
from bigdl.optim.optimizer import SeveralIteration
from zoo.util.triggers import SeveralIteration as ZSeveralIteration
from zoo.util.triggers import MinLoss as ZMinLoss
from zoo.util.triggers import TriggerAnd as ZTriggerAnd
est = Estimator.from_keras(keras_model=model)
OrcaContext.train_data_store = "DISK_2"
with self.assertRaises(Exception) as context:
est.fit(data=data_shard,
batch_size=4,
epochs=10,
validation_data=data_shard,
checkpoint_trigger=SeveralIteration(2))
self.assertTrue('Please use a trigger defined in zoo.util.triggers' in
str(context.exception))
est.fit(data=data_shard,
batch_size=4,
epochs=10,
validation_data=data_shard,
checkpoint_trigger=ZTriggerAnd(ZSeveralIteration(2),
ZMinLoss(0.2)))
OrcaContext.train_data_store = "DRAM"
def test_estimator_keras_with_bigdl_optim_method(self):
tf.reset_default_graph()
model = self.create_model()
dataset = tf.data.Dataset.from_tensor_slices(
(np.random.randint(0, 200, size=(100, 1)),
np.random.randint(0, 50, size=(100, 1)),
np.ones(shape=(100, ), dtype=np.int32)))
dataset = dataset.map(lambda user, item, label: [(user, item), label])
from zoo.orca.learn.optimizers import SGD
from zoo.orca.learn.optimizers.schedule import Plateau
sgd = SGD(learningrate=0.1,
learningrate_schedule=Plateau(
"score",
factor=0.1,
patience=10,
mode="min",
))
est = Estimator.from_keras(keras_model=model, optimizer=sgd)
est.fit(data=dataset, batch_size=8, epochs=10, validation_data=dataset)
def test_estimator_keras_learning_rate_schedule(self):
tf.reset_default_graph()
# loss = reduce_sum(w)
# dloss/dw = 1
model = self.create_model_lr_schedule(0.1, 1, 0.1)
dataset = tf.data.Dataset.from_tensor_slices((np.ones(
(16, 8)), np.zeros((16, 1))))
est = Estimator.from_keras(keras_model=model)
weights_before = model.get_weights()[0]
est.fit(data=dataset, batch_size=8, epochs=1, validation_data=dataset)
sess = tf.keras.backend.get_session()
iteartion = sess.run(model.optimizer.iterations)
weights_after = model.get_weights()[0]
first_step = weights_before - 0.1
second_step = first_step - 0.01
assert iteartion == 2
assert np.allclose(second_step, weights_after)
base_model.trainable = False
base_model.summary()
model = tf.keras.Sequential([
base_model,
keras.layers.GlobalAveragePooling2D(),
keras.layers.Dense(1, activation='sigmoid')
])
model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.0001),
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
len(model.trainable_variables)
epochs = 3
est = Estimator.from_keras(keras_model=model)
est.fit(train_dataset,
batch_size=batch_size,
epochs=epochs,
validation_data=validation_dataset
)
result = est.evaluate(validation_dataset)
print(result)
base_model.trainable = True
# Let's take a look to see how many layers are in the base model
print("Number of layers in the base model: ", len(base_model.layers))
# Fine tune from this layer onwards
fine_tune_at = 100
def main(cluster_mode, max_epoch, file_path, batch_size, platform,
non_interactive):
import matplotlib
if not non_interactive and platform == "mac":
matplotlib.use('qt5agg')
if cluster_mode == "local":
init_orca_context(cluster_mode="local", cores=4, memory="3g")
elif cluster_mode == "yarn":
init_orca_context(cluster_mode="yarn-client",
num_nodes=2,
cores=2,
driver_memory="3g")
load_data(file_path)
img_dir = os.path.join(file_path, "train")
label_dir = os.path.join(file_path, "train_masks")
# Here we only take the first 1000 files for simplicity
df_train = pd.read_csv(os.path.join(file_path, 'train_masks.csv'))
ids_train = df_train['img'].map(lambda s: s.split('.')[0])
ids_train = ids_train[:1000]
x_train_filenames = []
y_train_filenames = []
for img_id in ids_train:
x_train_filenames.append(os.path.join(img_dir,
"{}.jpg".format(img_id)))
y_train_filenames.append(
os.path.join(label_dir, "{}_mask.gif".format(img_id)))
x_train_filenames, x_val_filenames, y_train_filenames, y_val_filenames = \
train_test_split(x_train_filenames, y_train_filenames, test_size=0.2, random_state=42)
def load_and_process_image(path):
array = mpimg.imread(path)
result = np.array(Image.fromarray(array).resize(size=(128, 128)))
result = result.astype(float)
result /= 255.0
return result
def load_and_process_image_label(path):
array = mpimg.imread(path)
result = np.array(Image.fromarray(array).resize(size=(128, 128)))
result = np.expand_dims(result[:, :, 1], axis=-1)
result = result.astype(float)
result /= 255.0
return result
train_images = np.stack(
[load_and_process_image(filepath) for filepath in x_train_filenames])
train_label_images = np.stack([
load_and_process_image_label(filepath)
for filepath in y_train_filenames
])
val_images = np.stack(
[load_and_process_image(filepath) for filepath in x_val_filenames])
val_label_images = np.stack([
load_and_process_image_label(filepath) for filepath in y_val_filenames
])
train_shards = XShards.partition({
"x": train_images,
"y": train_label_images
})
val_shards = XShards.partition({"x": val_images, "y": val_label_images})
# Build the U-Net model
def conv_block(input_tensor, num_filters):
encoder = layers.Conv2D(num_filters, (3, 3),
padding='same')(input_tensor)
encoder = layers.Activation('relu')(encoder)
encoder = layers.Conv2D(num_filters, (3, 3), padding='same')(encoder)
encoder = layers.Activation('relu')(encoder)
return encoder
def encoder_block(input_tensor, num_filters):
encoder = conv_block(input_tensor, num_filters)
encoder_pool = layers.MaxPooling2D((2, 2), strides=(2, 2))(encoder)
return encoder_pool, encoder
def decoder_block(input_tensor, concat_tensor, num_filters):
decoder = layers.Conv2DTranspose(num_filters, (2, 2),
strides=(2, 2),
padding='same')(input_tensor)
decoder = layers.concatenate([concat_tensor, decoder], axis=-1)
decoder = layers.Activation('relu')(decoder)
decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
decoder = layers.Activation('relu')(decoder)
decoder = layers.Conv2D(num_filters, (3, 3), padding='same')(decoder)
decoder = layers.Activation('relu')(decoder)
return decoder
inputs = layers.Input(shape=(128, 128, 3)) # 128
encoder0_pool, encoder0 = encoder_block(inputs, 16) # 64
encoder1_pool, encoder1 = encoder_block(encoder0_pool, 32) # 32
encoder2_pool, encoder2 = encoder_block(encoder1_pool, 64) # 16
encoder3_pool, encoder3 = encoder_block(encoder2_pool, 128) # 8
center = conv_block(encoder3_pool, 256) # center
decoder3 = decoder_block(center, encoder3, 128) # 16
decoder2 = decoder_block(decoder3, encoder2, 64) # 32
decoder1 = decoder_block(decoder2, encoder1, 32) # 64
decoder0 = decoder_block(decoder1, encoder0, 16) # 128
outputs = layers.Conv2D(1, (1, 1), activation='sigmoid')(decoder0)
net = models.Model(inputs=[inputs], outputs=[outputs])
# Define custom metrics
def dice_coeff(y_true, y_pred):
smooth = 1.
# Flatten
y_true_f = tf.reshape(y_true, [-1])
y_pred_f = tf.reshape(y_pred, [-1])
intersection = tf.reduce_sum(y_true_f * y_pred_f)
score = (2. * intersection + smooth) / \
(tf.reduce_sum(y_true_f) + tf.reduce_sum(y_pred_f) + smooth)
return score
# Define custom loss function
def dice_loss(y_true, y_pred):
loss = 1 - dice_coeff(y_true, y_pred)
return loss
def bce_dice_loss(y_true, y_pred):
loss = losses.binary_crossentropy(y_true, y_pred) + dice_loss(
y_true, y_pred)
return loss
# compile model
net.compile(optimizer=tf.keras.optimizers.Adam(2e-3), loss=bce_dice_loss)
print(net.summary())
# create an estimator from keras model
est = Estimator.from_keras(keras_model=net)
# fit with estimator
est.fit(data=train_shards, batch_size=batch_size, epochs=max_epoch)
# evaluate with estimator
result = est.evaluate(val_shards)
print(result)
# predict with estimator
val_shards.cache()
val_image_shards = val_shards.transform_shard(
lambda val_dict: {"x": val_dict["x"]})
pred_shards = est.predict(data=val_image_shards, batch_size=batch_size)
pred = pred_shards.collect()[0]["prediction"]
val_image_label = val_shards.collect()[0]
val_image = val_image_label["x"]
val_label = val_image_label["y"]
if not non_interactive:
# visualize 5 predicted results
plt.figure(figsize=(10, 20))
for i in range(5):
img = val_image[i]
label = val_label[i]
predicted_label = pred[i]
plt.subplot(5, 3, 3 * i + 1)
plt.imshow(img)
plt.title("Input image")
plt.subplot(5, 3, 3 * i + 2)
plt.imshow(label[:, :, 0], cmap='gray')
plt.title("Actual Mask")
plt.subplot(5, 3, 3 * i + 3)
plt.imshow(predicted_label, cmap='gray')
plt.title("Predicted Mask")
plt.suptitle("Examples of Input Image, Label, and Prediction")
plt.show()
stop_orca_context()
validationDF = ncf_features.genData(tDF, sc, spark, args.validation_start,
args.validation_end, neg_rate,
sliding_length, u_limit, m_limit)
validationDF.show(5)
testDF = ncf_features.genData(tDF, sc, spark, args.test_start,
args.test_end, neg_rate, sliding_length,
u_limit, m_limit)
#testDF.show(5)
inferenceDF = ncf_features.genData(tDF, sc, spark, args.inference_start,
args.inference_end, neg_rate,
sliding_length, u_limit, m_limit)
#inferenceDF.show(5)
model = ncf_model.getKerasModel(u_limit, m_limit, u_output, m_output,
args.log_dir)
est = Estimator.from_keras(model, model_dir=args.log_dir)
est.fit(data=trainingDF,
batch_size=batch_size,
epochs=max_epoch,
feature_cols=['features'],
label_cols=['labels'],
validation_data=validationDF)
# save the model
est.save_keras_model(save_model_dir)
# metrics ,result and save model
print(model.metrics_names)
#Orca the predict function supports native spark data frame ! Just need to tell batch_size and feature_cols
# use a new Estimamtor to validate load model API
pre_est = Estimator.load_keras_model(save_model_dir)
prediction_df = pre_est.predict(inferenceDF,
batch_size=batch_size,
