def compare_layer(self,
klayer,
zlayer,
input_data,
weight_converter=None,
is_training=False,
rtol=1e-6,
atol=1e-6):
"""
Compare forward results for Keras layer against Zoo Keras API layer.
"""
from keras.models import Sequential as KSequential
from zoo.pipeline.api.keras.models import Sequential as ZSequential
zmodel = ZSequential()
zmodel.add(zlayer)
kmodel = KSequential()
kmodel.add(klayer)
koutput = kmodel.predict(input_data)
from zoo.pipeline.api.keras.layers import BatchNormalization
if isinstance(zlayer, BatchNormalization):
k_running_mean = K.eval(klayer.running_mean)
k_running_std = K.eval(klayer.running_std)
zlayer.set_running_mean(k_running_mean)
zlayer.set_running_std(k_running_std)
if kmodel.get_weights():
zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
zmodel.training(is_training)
zoutput = zmodel.forward(input_data)
self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)
def compare_unary_op(self, kk_func, z_layer, shape, rtol=1e-5, atol=1e-5):
x = klayers.Input(shape=shape[1:])
batch = shape[0]
kkresult = kk_func(x)
x_value = np.random.uniform(0, 1, shape)
k_grads = KK.get_session().run(KK.gradients(kkresult, x),
feed_dict={x: x_value})
k_output = KK.get_session().run(kkresult, feed_dict={x: x_value})
model = Sequential()
model.add(InputLayer(shape[1:]))
model.add(z_layer)
z_output = model.forward(x_value)
grad_output = np.array(z_output)
grad_output.fill(1.0)
z_grad = model.backward(x_value, grad_output)
z_output2 = model.forward(x_value)
z_grad2 = model.backward(x_value, grad_output)
self.assert_allclose(z_output, z_output2, rtol, atol)
self.assert_allclose(z_grad, z_grad2, rtol, atol)
self.assert_allclose(z_output, k_output, rtol, atol)
self.assert_allclose(z_grad, k_grads[0], rtol, atol)
def init_with_default_embedding(cls, vocab=40990, seq_len=77, n_block=12, resid_drop=0.1,
attn_drop=0.1, n_head=12, hidden_size=768,
embedding_drop=0.1, mask_attention=True):
"""
vocab: vocabulary size of training data, default is 40990
seq_len: max sequence length of training data, default is 77
n_block: block number, default is 12
resid_drop: drop probability of projection, default is 0.1
attn_drop: drop probability of attention, default is 0.1
n_head: head number, default is 12
hidden_size: is also embedding size
embedding_drop: drop probability of embedding layer, default is 0.1
mask_attention: whether unidirectional or bidirectional, default is true(unidirectional)
"""
from bigdl.nn.layer import Squeeze
embedding = Sequential()
embedding.add(Reshape([seq_len * 2], input_shape=(seq_len, 2)))\
.add(Embedding(vocab, hidden_size, input_length=seq_len * 2))\
.add(Dropout(embedding_drop))\
.add(Reshape((seq_len, 2, hidden_size)))\
.add(KerasLayerWrapper(Sum(dimension=3, squeeze=True)))
# walk around for bug #1208, need remove this line after the bug fixed
embedding.add(KerasLayerWrapper(Squeeze(dim=3)))
return TransformerLayer(n_block, resid_drop, attn_drop, n_head, mask_attention,
embedding, input_shape=(seq_len, 2))
def test_square_as_first_layer(self):
def z_func(x):
return square(x)
ll = Lambda(function=z_func, input_shape=[2, 3])
seq = Sequential()
seq.add(ll)
result = seq.forward(np.ones([2, 3]))
assert (result == np.ones([2, 3])).all()
def test_save_load_Sequential(self):
zmodel = ZSequential()
dense = ZLayer.Dense(10, input_dim=5)
zmodel.add(dense)
tmp_path = create_tmp_path()
zmodel.saveModel(tmp_path, None, True)
model_reloaded = Net.load(tmp_path)
input_data = np.random.random([10, 5])
y = np.random.random([10, 10])
model_reloaded.compile(optimizer="adam", loss="mse")
model_reloaded.fit(x=input_data, y=y, batch_size=8, nb_epoch=1)
def test_regularizer(self):
model = ZSequential()
model.add(
ZLayer.Dense(16,
W_regularizer=regularizers.l2(0.001),
activation='relu',
input_shape=(10000, )))
model.summary()
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc'])
def init(cls,
vocab=40990,
seq_len=77,
n_block=12,
hidden_drop=0.1,
attn_drop=0.1,
n_head=12,
hidden_size=768,
embedding_drop=0.1,
initializer_range=0.02,
bidirectional=False,
output_all_block=False):
"""
vocab: vocabulary size of training data, default is 40990
seq_len: max sequence length of training data, default is 77
n_block: block number, default is 12
hidden_drop: drop probability of projection, default is 0.1
attn_drop: drop probability of attention, default is 0.1
n_head: head number, default is 12
hidden_size: is also embedding size
embedding_drop: drop probability of embedding layer, default is 0.1
initializer_range: weight initialization range, default is 0.02
bidirectional: whether unidirectional or bidirectional, default is unidirectional
output_all_block: whether output all blocks' output
"""
if hidden_size < 0:
raise TypeError(
'hidden_size must be greater than 0 with default embeddding layer'
)
from bigdl.nn.layer import Squeeze
word_input = InputLayer(input_shape=(seq_len, ))
postion_input = InputLayer(input_shape=(seq_len, ))
embedding = Sequential()
embedding.add(Merge(layers=[word_input, postion_input], mode='concat'))\
.add(Reshape([seq_len * 2]))\
.add(Embedding(vocab, hidden_size, input_length=seq_len * 2,
weights=np.random.normal(0.0, initializer_range, (vocab, hidden_size))))\
.add(Dropout(embedding_drop))\
.add(Reshape((seq_len, 2, hidden_size)))\
.add(KerasLayerWrapper(Sum(dimension=3, squeeze=True)))
# walk around for bug #1208, need remove this line after the bug fixed
embedding.add(KerasLayerWrapper(Squeeze(dim=3)))
shape = ((seq_len, ), (seq_len, ))
return TransformerLayer(n_block,
hidden_drop,
attn_drop,
n_head,
initializer_range,
bidirectional,
output_all_block,
embedding,
input_shape=shape)
def _build_model(sequence_length):
model = Sequential()
model.add(Embedding(20, 10, input_length=sequence_length))
model.add(Convolution1D(4, 3))
model.add(Flatten())
model.add(Dense(5, activation="softmax"))
return model
def build_model(self):
model = Sequential()
model.add(Dense(24, input_dim=self.state_size, activation='relu'))
model.add(Dense(24, activation='relu'))
model.add(Dense(self.action_size, activation='linear'))
model.summary()
model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate))
return model
def build_model(self):
model = Sequential()
model.add(InputLayer(input_shape=self.feature_shape)) \
.add(LSTM(input_shape=self.feature_shape, output_dim=self.hidden_layers[0],
return_sequences=True))
for ilayer in range(1, len(self.hidden_layers) - 1):
model.add(LSTM(output_dim=self.hidden_layers[ilayer], return_sequences=True)) \
.add(Dropout(self.dropouts[ilayer]))
model.add(LSTM(self.hidden_layers[-1], return_sequences=False)) \
.add(Dropout(self.dropouts[-1]))
model.add(Dense(output_dim=1))
return model
def test_merge_method_seq_concat(self):
zx1 = ZLayer.Input(shape=(10, ))
zx2 = ZLayer.Input(shape=(10, ))
zy1 = ZLayer.Dense(12, activation="sigmoid")(zx1)
zbranch1_node = ZModel(zx1, zy1)(zx1)
zbranch2 = ZSequential()
zbranch2.add(ZLayer.Dense(12, input_dim=10))
zbranch2_node = zbranch2(zx2)
zz = ZLayer.merge([zbranch1_node, zbranch2_node], mode="concat")
zmodel = ZModel([zx1, zx2], zz)
kx1 = KLayer.Input(shape=(10, ))
kx2 = KLayer.Input(shape=(10, ))
ky1 = KLayer.Dense(12, activation="sigmoid")(kx1)
kbranch1_node = KModel(kx1, ky1)(kx1)
kbranch2 = KSequential()
kbranch2.add(KLayer.Dense(12, input_dim=10))
kbranch2_node = kbranch2(kx2)
kz = KLayer.merge([kbranch1_node, kbranch2_node], mode="concat")
kmodel = KModel([kx1, kx2], kz)
input_data = [np.random.random([2, 10]), np.random.random([2, 10])]
self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)
def predict(model_path, image_path, top_n):
sc = init_nncontext(
"Image classification inference example using int8 quantized model")
images = ImageSet.read(image_path, sc, image_codec=1)
model = ImageClassifier.load_model(model_path)
output = model.predict_image_set(images)
label_map = model.get_config().label_map()
# list of images composing uri and results in tuple format
predicts = output.get_predict().collect()
sequential = Sequential()
sequential.add(Activation("softmax", input_shape=predicts[0][1][0].shape))
for pre in predicts:
(uri, probs) = pre
out = sequential.forward(probs[0])
sortedProbs = [(prob, index) for index, prob in enumerate(out)]
sortedProbs.sort()
print("Image : %s, top %d prediction result" % (uri, top_n))
for i in range(top_n):
print(
"\t%s, %f" %
(label_map[sortedProbs[999 - i][1]], sortedProbs[999 - i][0]))
def test_merge_method_seq_concat(self):
zx1 = ZLayer.Input(shape=(10, ))
zx2 = ZLayer.Input(shape=(10, ))
zy1 = ZLayer.Dense(12, activation="sigmoid")(zx1)
zbranch1_node = ZModel(zx1, zy1)(zx1)
zbranch2 = ZSequential()
zbranch2.add(ZLayer.Dense(12, input_dim=10))
zbranch2_node = zbranch2(zx2)
zz = ZLayer.merge([zbranch1_node, zbranch2_node], mode="concat")
zmodel = ZModel([zx1, zx2], zz)
kx1 = KLayer.Input(shape=(10, ))
kx2 = KLayer.Input(shape=(10, ))
ky1 = KLayer.Dense(12, activation="sigmoid")(kx1)
kbranch1_node = KModel(kx1, ky1)(kx1)
kbranch2 = KSequential()
kbranch2.add(KLayer.Dense(12, input_dim=10))
kbranch2_node = kbranch2(kx2)
kz = KLayer.merge([kbranch1_node, kbranch2_node], mode="concat")
kmodel = KModel([kx1, kx2], kz)
input_data = [np.random.random([2, 10]), np.random.random([2, 10])]
self.compare_layer(kmodel, zmodel, input_data, self.convert_two_dense)
def compare_layer(self, klayer, zlayer, input_data, weight_converter=None,
is_training=False, rtol=1e-6, atol=1e-6):
"""
Compare forward results for Keras layer against Zoo Keras API layer.
"""
from keras.models import Sequential as KSequential
from zoo.pipeline.api.keras.models import Sequential as ZSequential
zmodel = ZSequential()
zmodel.add(zlayer)
kmodel = KSequential()
kmodel.add(klayer)
koutput = kmodel.predict(input_data)
from zoo.pipeline.api.keras.layers import BatchNormalization
if isinstance(zlayer, BatchNormalization):
k_running_mean = K.eval(klayer.running_mean)
k_running_std = K.eval(klayer.running_std)
zlayer.set_running_mean(k_running_mean)
zlayer.set_running_std(k_running_std)
if kmodel.get_weights():
zmodel.set_weights(weight_converter(klayer, kmodel.get_weights()))
zmodel.training(is_training)
zoutput = zmodel.forward(input_data)
self.assert_allclose(zoutput, koutput, rtol=rtol, atol=atol)
def build_model(self):
model = Sequential()
model.add(
InputLayer(input_shape=(self.sequence_length, self.token_length)))
if self.encoder.lower() == 'cnn':
model.add(
Convolution1D(self.encoder_output_dim, 5, activation='relu'))
model.add(GlobalMaxPooling1D())
elif self.encoder.lower() == 'lstm':
model.add(LSTM(self.encoder_output_dim))
elif self.encoder.lower() == 'gru':
model.add(GRU(self.encoder_output_dim))
else:
raise ValueError('Unsupported encoder: ' + self.encoder)
model.add(Dense(128))
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(self.class_num, activation='softmax'))
return model
def buildmodel():
print("Now we build the model")
model = Sequential()
model.add(
Convolution2D(32,
8,
8,
subsample=(4, 4),
border_mode='same',
input_shape=(img_rows, img_cols,
img_channels))) # 80*80*4
model.add(Activation('relu'))
model.add(Convolution2D(64, 4, 4, subsample=(2, 2), border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3, subsample=(1, 1), border_mode='same'))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dense(2))
model.compile(loss='mse', optimizer='adam')
print("We finish building the model")
return model
def _build_mode(self):
print("Now we build the model")
model = Sequential()
model.add(Convolution2D(32, 8, 8, subsample=(4, 4), border_mode='same',
input_shape=(IMAGE_ROWS, IMAGE_COLS, IMAGE_CHANNELS))) # 80*80*4
model.add(Activation('relu'))
model.add(Convolution2D(64, 4, 4, subsample=(2, 2), border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3, subsample=(1, 1), border_mode='same'))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dense(2))
# get the 1 * 2 output represent each action's probability
model.add(Activation('softmax'))
return model
x_train = unroll(x_train,unroll_length)
x_test = unroll(x_test,unroll_length)
y_train = y_train[-x_train.shape[0]:]
y_test = y_test[-x_test.shape[0]:]
# see the shape
print("x_train", x_train.shape)
print("y_train", y_train.shape)
print("x_test", x_test.shape)
print("y_test", y_test.shape)
# Build the model
model = Sequential()
model.add(LSTM(
input_shape=(x_train.shape[1], x_train.shape[-1]),
output_dim=20,
return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(
10,
return_sequences=False))
model.add(Dropout(0.2))
model.add(Dense(
output_dim=1))
model.compile(loss='mse', optimizer='rmsprop')
%%time
# Train the model
df = pd.read_csv("../resources/datasets/dataset-1_converted.csv")
trainDf, testDf = train_test_split(df, test_size=0.2)
print("Created Train and Test Df\n")
predictionColumn = 'slotOccupancy'
x = trainDf.drop(columns=[predictionColumn])
inputs = len(x.columns)
y = trainDf[[predictionColumn]]
outputs = len(y.columns)
model = Sequential()
model.add(Dense(output_dim=inputs, activation="relu", input_shape=(inputs, )))
model.add(Dense(output_dim=inputs, activation="relu"))
model.add(Dense(output_dim=outputs))
model.compile(optimizer="adam", loss="mean_squared_error")
model.summary()
print("Created Sequential Model!\n")
xNumpy = x.to_numpy()
yNumpy = y.to_numpy()
# model.fit(x=xNumpy, y=yNumpy, nb_epoch=1, distributed=False)
import tensorflow as tf
weights = np.array(model.get_weights(), dtype=object)
def build_model(self):
model = Sequential()
model.add(InputLayer(input_shape=(self.sequence_length, self.token_length)))
if self.encoder.lower() == 'cnn':
model.add(Convolution1D(self.encoder_output_dim, 5, activation='relu'))
model.add(GlobalMaxPooling1D())
elif self.encoder.lower() == 'lstm':
model.add(LSTM(self.encoder_output_dim))
elif self.encoder.lower() == 'gru':
model.add(GRU(self.encoder_output_dim))
else:
raise ValueError('Unsupported encoder: ' + self.encoder)
model.add(Dense(128))
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(self.class_num, activation='softmax'))
return model
train_rdd = sc.parallelize(t_train_img).zip(sc.parallelize(
train_lbl)).map(lambda (feature, label): Sample.from_ndarray(
feature, label + 1) # 如果用 keras.fit 则需要 -1.
)
test_rdd = sc.parallelize(t_test_img).zip(sc.parallelize(test_lbl)).map(
lambda (feature, label): Sample.from_ndarray(feature, label + 1))
# 用 Zoo-Keras 定义模型的网络结构.
input_shape = (NUM_CLASS_LABEL, NUM_IMAGE_CHANNEL, IMAGE_SIZE, IMAGE_SIZE)
both_input = Input(shape=input_shape)
convolve_net = Sequential()
convolve_net.add(
Convolution2D(
nb_filter=LAYER_1_NUM_CHANNEL, # 通道: 4 -> 8.
nb_row=CONVOLVE_1_KERNEL_SIZE, # 尺寸: 32 - 9 + 1 = 24
nb_col=CONVOLVE_1_KERNEL_SIZE,
activation="relu",
input_shape=(NUM_IMAGE_CHANNEL, IMAGE_SIZE, IMAGE_SIZE),
W_regularizer=L2Regularizer(args.penalty_rate)))
convolve_net.add(
AveragePooling2D(
pool_size=(
POOLING_1_WINDOW_SIZE, # 尺寸: 24 / 2 = 12.
POOLING_1_WINDOW_SIZE),
strides=(POOLING_1_STRIDE_SIZE, POOLING_1_STRIDE_SIZE)))
convolve_net.add(BatchNormalization())
convolve_net.add(
Convolution2D(
nb_filter=LAYER_2_NUM_CHANNEL, # 通道: 8 -> 2.
nb_row=CONVOLVE_2_KERNEL_SIZE, # 尺寸: 12 - 5 + 1 = 8.
nb_col=CONVOLVE_2_KERNEL_SIZE,
def build_model(self):
model = Sequential()
model.add(self.embedding)
if self.encoder.lower() == 'cnn':
model.add(Convolution1D(self.encoder_output_dim, 5, activation='relu'))
model.add(GlobalMaxPooling1D())
elif self.encoder.lower() == 'lstm':
model.add(LSTM(self.encoder_output_dim))
elif self.encoder.lower() == 'gru':
model.add(GRU(self.encoder_output_dim))
else:
raise ValueError('Unsupported encoder for TextClassifier: ' + self.encoder)
model.add(Dense(128))
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(self.class_num, activation='softmax'))
return model
