def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
input_tensor1 = Input(shape=(3, 32, 32), dtype="float32", name="input1")
input_tensor2 = Input(shape=(3, 32, 32), dtype="float32", name="input2")
ot1 = cifar_cnn_sub(input_tensor1, 1)
model1 = Model(input_tensor1, ot1)
print(model1.summary())
ot2 = cifar_cnn_sub(input_tensor2, 2)
model2 = Model(input_tensor2, ot2)
print(model2.summary())
output_tensor = Concatenate(axis=1)([model1.output, model2.output])
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_0_4")(output_tensor)
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = Dense(512, activation="relu")(output_tensor)
output_tensor = Dense(num_classes)(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
model = Model([input_tensor1, input_tensor2], output_tensor)
# print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit([x_train, x_train],
y_train,
epochs=160,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_CNN),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)
])
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
#x_train *= 0
#y_train = np.random.randint(1, 9, size=(num_samples,1), dtype='int32')
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
model = Sequential()
model.add(Conv2D(filters=32, input_shape=(3,32,32), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Flatten())
model.add(Dense(512, activation="relu"))
model.add(Dense(num_classes))
model.add(Activation("softmax"))
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit(x_train, y_train, epochs=1)
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
model = Sequential()
model.add(Conv2D(filters=32, input_shape=(3,32,32), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Flatten())
model.add(Dense(512, activation="relu"))
model.add(Dense(num_classes))
model.add(Activation("softmax"))
opt = flexflow.keras.optimizers.SGD(learning_rate=0.02)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit(x_train, y_train, epochs=80, callbacks=[VerifyMetrics(ModelAccuracy.CIFAR10_CNN), EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)])
def cifar_cnn_concat():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
#x_train *= 0
#y_train = np.random.randint(1, 9, size=(num_samples,1), dtype='int32')
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
input_tensor1 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
input_tensor2 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
ot1 = cifar_cnn_sub(input_tensor1, 1)
ot2 = cifar_cnn_sub(input_tensor2, 2)
ot3 = cifar_cnn_sub(input_tensor2, 3)
output_tensor = Concatenate(axis=1)([ot1, ot2, ot3])
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
o1 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_0_4")(output_tensor)
o2 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_1_4")(output_tensor)
output_tensor = Concatenate(axis=1)([o1, o2])
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = Dense(512, activation="relu")(output_tensor)
output_tensor = Dense(num_classes)(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
model = Model([input_tensor1, input_tensor2], output_tensor)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit([x_train, x_train], y_train, epochs=1)
def top_level_task():
num_classes = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
print("shape: ", x_train.shape, x_train.__array_interface__["strides"])
layers = [
Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu"),
Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu"),
MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="valid"),
Flatten()
]
model1 = Sequential(layers)
input_tensor = Input(shape=(12544, ), dtype="float32")
output = Dense(512, input_shape=(12544, ), activation="relu")(input_tensor)
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
model2 = Model(input_tensor, output)
model = Sequential()
model.add(model1)
model.add(model2)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit(x_train,
y_train,
epochs=5,
callbacks=[
VerifyMetrics(ModelAccuracy.MNIST_CNN),
EpochVerifyMetrics(ModelAccuracy.MNIST_CNN)
])
def top_level_task():
num_classes = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
input_tensor = Input(shape=(1, 28, 28), dtype="float32")
output = Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor)
# output = Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu")(output)
output = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
kernel_initializer=GlorotUniform(123),
bias_initializer=Zeros())(output)
output = Activation('relu')(output)
output = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding="valid")(output)
output = Dropout(0.25)(output)
output = Flatten()(output)
output = Dense(128, activation="relu")(output)
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
model = Model(input_tensor, output)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
flatten1 = model.get_layer(name='flat')
t1 = flatten1.output_tensors[0]
t2 = flatten1.input_tensors[0]
print("t1: ", t1.to_layers, " ", t1.from_layer)
print("t2: ", t2.to_layers, " ", t2.from_layer)
model.fit(x_train,
y_train,
epochs=5,
callbacks=[
VerifyMetrics(ModelAccuracy.MNIST_CNN),
EpochVerifyMetrics(ModelAccuracy.MNIST_CNN)
])
def top_level_task():
num_classes = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
print("shape: ", x_train.shape, x_train.__array_interface__["strides"])
# model = Sequential()
# model.add(Conv2D(filters=32, input_shape=(1,28,28), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
# model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
# model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
# model.add(Flatten())
# model.add(Dense(128, activation="relu"))
# model.add(Dense(num_classes))
# model.add(Activation("softmax"))
layers = [
Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu"),
Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu"),
MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="valid"),
Flatten(),
Dense(128, activation="relu"),
Dense(num_classes),
Activation("softmax")
]
model = Sequential(layers)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit(x_train, y_train, epochs=1)
def cnn_concat():
num_classes = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
input_tensor = Input(batch_shape=[0, 1, 28, 28], dtype="float32")
t1 = Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor)
t2 = Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor)
output = Concatenate(axis=1)([t1, t2])
output = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding="valid")(output)
output = Flatten()(output)
output = Dense(128, activation="relu")(output)
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
model = Model(input_tensor, output)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit(x_train, y_train, epochs=1)
def create_teacher_model_cnn(num_classes, x_train, y_train):
model = Sequential()
model.add(Conv2D(filters=32, input_shape=(1,28,28), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Flatten())
model.add(Dense(128, activation="relu"))
model.add(Dense(num_classes))
model.add(Activation("softmax"))
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit(x_train, y_train, epochs=5)
return model
def create_student_model_cnn(teacher_model, num_classes, x_train, y_train):
conv1 = teacher_model.get_layer(0)
c1_kernel, c1_bias = conv1.get_weights(teacher_model.ffmodel)
print(c1_kernel.shape, c1_bias.shape)
conv2 = teacher_model.get_layer(1)
c2_kernel, c2_bias = conv2.get_weights(teacher_model.ffmodel)
dense1 = teacher_model.get_layer(4)
d1_kernel, d1_bias = dense1.get_weights(teacher_model.ffmodel)
dense2 = teacher_model.get_layer(5)
d2_kernel, d2_bias = dense2.get_weights(teacher_model.ffmodel)
model = Sequential()
model.add(Conv2D(filters=32, input_shape=(1,28,28), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Flatten())
model.add(Dense(128, activation="relu"))
model.add(Dense(num_classes))
model.add(Activation("softmax"))
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
conv1s = model.get_layer(0)
conv2s = model.get_layer(1)
dense1s = model.get_layer(4)
dense2s = model.get_layer(5)
conv1s.set_weights(model.ffmodel, c1_kernel, c1_bias)
conv2s.set_weights(model.ffmodel, c2_kernel, c2_bias)
dense1s.set_weights(model.ffmodel, d1_kernel, d1_bias)
dense2s.set_weights(model.ffmodel, d2_kernel, d2_bias)
print(model.summary())
model.fit(x_train, y_train, epochs=1)
def create_student_model_cnn(teacher_model, num_classes, x_train, y_train):
conv1 = teacher_model.get_layer(index=0)
c1_kernel, c1_bias = conv1.get_weights(teacher_model.ffmodel)
print(c1_kernel.shape, c1_bias.shape)
conv2 = teacher_model.get_layer(index=1)
c2_kernel, c2_bias = conv2.get_weights(teacher_model.ffmodel)
dense1 = teacher_model.get_layer(index=4)
d1_kernel, d1_bias = dense1.get_weights(teacher_model.ffmodel)
dense2 = teacher_model.get_layer(index=5)
d2_kernel, d2_bias = dense2.get_weights(teacher_model.ffmodel)
model = Sequential()
model.add(Conv2D(filters=32, input_shape=(1,28,28), kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(Conv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding=(1,1), activation="relu"))
model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding="valid"))
model.add(Flatten())
model.add(Dense(128, activation="relu", name="dense1"))
model.add(Dense(num_classes))
model.add(Activation("softmax"))
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy', 'sparse_categorical_crossentropy'])
conv1s = model.get_layer(index=0)
conv2s = model.get_layer(index=1)
dense1s = model.get_layer(name="dense1")
dense2s = model.get_layer(index=5)
conv1s.set_weights(model.ffmodel, c1_kernel, c1_bias)
conv2s.set_weights(model.ffmodel, c2_kernel, c2_bias)
dense1s.set_weights(model.ffmodel, d1_kernel, d1_bias)
dense2s.set_weights(model.ffmodel, d2_kernel, d2_bias)
print(model.summary())
model.fit(x_train, y_train, epochs=5, callbacks=[VerifyMetrics(ModelAccuracy.MNIST_CNN), EpochVerifyMetrics(ModelAccuracy.MNIST_CNN)])
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
#teacher
input_tensor1 = Input(shape=(3, 32, 32), dtype="float32")
c1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
c2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
d1 = Dense(512, activation="relu")
d2 = Dense(num_classes)
output_tensor = c1(input_tensor1)
output_tensor = c2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="same")(output_tensor)
output_tensor = c3(output_tensor)
output_tensor = c4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = d1(output_tensor)
output_tensor = d2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
teacher_model = Model(input_tensor1, output_tensor)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
teacher_model.compile(
optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
teacher_model.fit(x_train, y_train, epochs=10)
c1_kernel, c1_bias = c1.get_weights(teacher_model.ffmodel)
c2_kernel, c2_bias = c2.get_weights(teacher_model.ffmodel)
c3_kernel, c3_bias = c3.get_weights(teacher_model.ffmodel)
c4_kernel, c4_bias = c4.get_weights(teacher_model.ffmodel)
d1_kernel, d1_bias = d1.get_weights(teacher_model.ffmodel)
d2_kernel, d2_bias = d2.get_weights(teacher_model.ffmodel)
#d2_kernel *= 0
c2_kernel_new = np.concatenate((c2_kernel, c2_kernel), axis=1)
print(c2_kernel.shape, c2_kernel_new.shape, c2_bias.shape)
#student model
input_tensor2 = Input(shape=(3, 32, 32), dtype="float32")
sc1_1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
sc1_2 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
sc2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sd1 = Dense(512, activation="relu")
sd2 = Dense(num_classes)
t1 = sc1_1(input_tensor2)
t2 = sc1_2(input_tensor2)
output_tensor = Concatenate(axis=1)([t1, t2])
output_tensor = sc2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="same")(output_tensor)
output_tensor = sc3(output_tensor)
output_tensor = sc4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = sd1(output_tensor)
output_tensor = sd2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
student_model = Model(input_tensor2, output_tensor)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
student_model.compile(
optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
sc1_1.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc1_2.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc2.set_weights(student_model.ffmodel, c2_kernel_new, c2_bias)
sc3.set_weights(student_model.ffmodel, c3_kernel, c3_bias)
sc4.set_weights(student_model.ffmodel, c4_kernel, c4_bias)
sd1.set_weights(student_model.ffmodel, d1_kernel, d1_bias)
sd2.set_weights(student_model.ffmodel, d2_kernel, d2_bias)
student_model.fit(x_train,
y_train,
epochs=160,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_CNN),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)
])
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
input_tensor1 = Input(shape=(3, 32, 32), dtype="float32")
output_tensor1 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor1)
output_tensor1 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor1)
output_tensor1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor1)
model1 = Model(input_tensor1, output_tensor1)
input_tensor2 = Input(shape=(3, 32, 32), dtype="float32")
output_tensor2 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor2)
output_tensor2 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor2)
output_tensor2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor2)
output_tensor2 = Flatten()(output_tensor2)
output_tensor2 = Dense(512, activation="relu")(output_tensor2)
output_tensor2 = Dense(num_classes)(output_tensor2)
output_tensor2 = Activation("softmax")(output_tensor2)
model2 = Model(input_tensor2, output_tensor2)
input_tensor3 = Input(shape=(3, 32, 32), dtype="float32")
output_tensor3 = model1(input_tensor3)
output_tensor3 = model2(output_tensor3)
model = Model(input_tensor3, output_tensor3)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit(x_train,
y_train,
epochs=40,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_CNN),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)
])
def top_level_task():
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
full_input_np = np.zeros((num_samples, 3, 229, 229), dtype=np.float32)
for i in range(0, num_samples):
image = x_train[i, :, :, :]
image = image.transpose(1, 2, 0)
pil_image = Image.fromarray(image)
pil_image = pil_image.resize((229, 229), Image.NEAREST)
image = np.array(pil_image, dtype=np.float32)
image = image.transpose(2, 0, 1)
full_input_np[i, :, :, :] = image
if (i == 0):
print(image)
full_input_np /= 255
y_train = y_train.astype('int32')
full_label_np = y_train
input_tensor = Input(shape=(3, 229, 229), dtype="float32")
output = Conv2D(filters=64,
input_shape=(3, 229, 229),
kernel_size=(11, 11),
strides=(4, 4),
padding=(2, 2),
activation="relu")(input_tensor)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Conv2D(filters=192,
kernel_size=(5, 5),
strides=(1, 1),
padding=(2, 2),
activation="relu")(output)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Flatten()(output)
output = Dense(4096, activation="relu")(output)
output = Dense(4096, activation="relu")(output)
output = Dense(10)(output)
output = Activation("softmax")(output)
model = Model(input_tensor, output)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit(full_input_np,
full_label_np,
epochs=160,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_ALEXNET),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_ALEXNET)
])
def cifar_cnn_net2net():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
#x_train *= 0
#y_train = np.random.randint(1, 9, size=(num_samples,1), dtype='int32')
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
#teacher
input_tensor1 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
c1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
d1 = Dense(512, activation="relu")
d2 = Dense(num_classes)
output_tensor = c1(input_tensor1)
output_tensor = c2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = c3(output_tensor)
output_tensor = c4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = d1(output_tensor)
output_tensor = d2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
teacher_model = Model(input_tensor1, output_tensor)
print(teacher_model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
teacher_model.compile(optimizer=opt)
teacher_model.fit(x_train, y_train, epochs=1)
c1_kernel, c1_bias = c1.get_weights(teacher_model.ffmodel)
c2_kernel, c2_bias = c2.get_weights(teacher_model.ffmodel)
c3_kernel, c3_bias = c3.get_weights(teacher_model.ffmodel)
c4_kernel, c4_bias = c4.get_weights(teacher_model.ffmodel)
d1_kernel, d1_bias = d1.get_weights(teacher_model.ffmodel)
d2_kernel, d2_bias = d2.get_weights(teacher_model.ffmodel)
#d2_kernel *= 0
c2_kernel_new = np.concatenate((c2_kernel, c2_kernel), axis=1)
print(c2_kernel.shape, c2_kernel_new.shape, c2_bias.shape)
#student model
input_tensor2 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
sc1_1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc1_2 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sd1 = Dense(512, activation="relu")
sd2 = Dense(num_classes)
t1 = sc1_1(input_tensor2)
t2 = sc1_2(input_tensor2)
output_tensor = Concatenate(axis=1)([t1, t2])
output_tensor = sc2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = sc3(output_tensor)
output_tensor = sc4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = sd1(output_tensor)
output_tensor = sd2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
student_model = Model(input_tensor2, output_tensor)
print(student_model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
student_model.compile(optimizer=opt)
sc1_1.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc1_2.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc2.set_weights(student_model.ffmodel, c2_kernel_new, c2_bias)
sc3.set_weights(student_model.ffmodel, c3_kernel, c3_bias)
sc4.set_weights(student_model.ffmodel, c4_kernel, c4_bias)
sd1.set_weights(student_model.ffmodel, d1_kernel, d1_bias)
sd2.set_weights(student_model.ffmodel, d2_kernel, d2_bias)
student_model.fit(x_train, y_train, epochs=1)
def cifar_alexnet():
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
full_input_np = np.zeros((num_samples, 3, 229, 229), dtype=np.float32)
for i in range(0, num_samples):
image = x_train[i, :, :, :]
image = image.transpose(1, 2, 0)
pil_image = Image.fromarray(image)
pil_image = pil_image.resize((229, 229), Image.NEAREST)
image = np.array(pil_image, dtype=np.float32)
image = image.transpose(2, 0, 1)
full_input_np[i, :, :, :] = image
if (i == 0):
print(image)
full_input_np /= 255
y_train = y_train.astype('int32')
full_label_np = y_train
input_tensor = Input(batch_shape=[0, 3, 229, 229], dtype="float32")
output = Conv2D(filters=64,
input_shape=(3, 229, 229),
kernel_size=(11, 11),
strides=(4, 4),
padding=(2, 2),
activation="relu")(input_tensor)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Conv2D(filters=192,
kernel_size=(5, 5),
strides=(1, 1),
padding=(2, 2),
activation="relu")(output)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Conv2D(filters=384,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = Conv2D(filters=256,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = MaxPooling2D(pool_size=(3, 3), strides=(2, 2),
padding="valid")(output)
output = Flatten()(output)
output = Dense(4096, activation="relu")(output)
output = Dense(4096, activation="relu")(output)
output = Dense(10)(output)
output = Activation("softmax")(output)
model = Model(input_tensor, output)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.001)
model.compile(optimizer=opt)
model.fit(full_input_np, full_label_np, epochs=1)