def cifar_cnn_concat():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
#x_train *= 0
#y_train = np.random.randint(1, 9, size=(num_samples,1), dtype='int32')
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
input_tensor1 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
input_tensor2 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
ot1 = cifar_cnn_sub(input_tensor1, 1)
ot2 = cifar_cnn_sub(input_tensor2, 2)
ot3 = cifar_cnn_sub(input_tensor2, 3)
output_tensor = Concatenate(axis=1)([ot1, ot2, ot3])
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
o1 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_0_4")(output_tensor)
o2 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_1_4")(output_tensor)
output_tensor = Concatenate(axis=1)([o1, o2])
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = Dense(512, activation="relu")(output_tensor)
output_tensor = Dense(num_classes)(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
model = Model([input_tensor1, input_tensor2], output_tensor)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit([x_train, x_train], y_train, epochs=1)
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
input_tensor1 = Input(shape=(3, 32, 32), dtype="float32", name="input1")
input_tensor2 = Input(shape=(3, 32, 32), dtype="float32", name="input2")
ot1 = cifar_cnn_sub(input_tensor1, 1)
model1 = Model(input_tensor1, ot1)
print(model1.summary())
ot2 = cifar_cnn_sub(input_tensor2, 2)
model2 = Model(input_tensor2, ot2)
print(model2.summary())
output_tensor = Concatenate(axis=1)([model1.output, model2.output])
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu",
name="conv2d_0_4")(output_tensor)
output_tensor = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = Dense(512, activation="relu")(output_tensor)
output_tensor = Dense(num_classes)(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
model = Model([input_tensor1, input_tensor2], output_tensor)
# print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit([x_train, x_train],
y_train,
epochs=160,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_CNN),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)
])
def top_level_task():
num_classes = 10
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
print("shape: ", x_train.shape)
# input_tensor1 = Input(shape=(256,))
input_tensor11 = Input(shape=(256,))
input_tensor12 = Input(shape=(256,))
input_tensor2 = Input(shape=(256,))
input_tensor3 = Input(shape=(256,))
input_tensor4 = Input(shape=(256,))
# t1 = Dense(512, activation="relu", name="dense1")(input_tensor1)
# t1 = Dense(512, activation="relu", name="dense12")(t1)
# model1 = Model(input_tensor1, t1)
t11 = Dense(512, activation="relu", name="dense1")(input_tensor11)
model11 = Model(input_tensor11, t11)
t12 = model11(input_tensor12)
t1 = Dense(512, activation="relu", name="dense12")(t12)
model1 = Model(input_tensor12, t1)
t2 = Dense(512, activation="relu", name="dense2")(input_tensor2)
t2 = Dense(512, activation="relu", name="dense22")(t2)
model2 = Model(input_tensor2, t2)
t3 = Dense(512, activation="relu", name="dense3")(input_tensor3)
t3 = Dense(512, activation="relu", name="dense33")(t3)
model3 = Model(input_tensor3, t3)
t4 = Dense(512, activation="relu", name="dense4")(input_tensor4)
t4 = Dense(512, activation="relu", name="dense44")(t4)
model4 = Model(input_tensor4, t4)
input_tensor = Input(shape=(784,))
t00 = Input(shape=(784,), name="input_00")
t01 = Input(shape=(784,), name="input_01")
t1 = model1(input_tensor)
t2 = model2(input_tensor)
t3 = model3(input_tensor)
t4 = model4(input_tensor)
output = Concatenate(axis=1)([t00, t01, t1, t2, t3, t4])
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
model = Model([t00, t01, input_tensor], output)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy', metrics=['accuracy', 'sparse_categorical_crossentropy'])
print(model.summary())
model.fit([x_train, x_train, x_train], y_train, epochs=10, callbacks=[VerifyMetrics(ModelAccuracy.MNIST_MLP), EpochVerifyMetrics(ModelAccuracy.MNIST_MLP)])
def build_model(loader, args, permanent_dropout=True, silent=False):
input_shapes = {}
dropout_rate = args.dropout
for fea_type, shape in loader.feature_shapes.items():
base_type = fea_type.split('.')[0]
if base_type in ['cell', 'drug']:
if not silent:
print('Feature encoding submodel for %s:', fea_type)
#box.summary(print_fn=logger.debug)
input_shapes[fea_type] = shape
inputs = []
encoded_inputs = []
for fea_name, fea_type in loader.input_features.items():
shape = loader.feature_shapes[fea_type]
fea_input = Input(shape, name='input.' + fea_name)
inputs.append(fea_input)
if fea_type in input_shapes:
if args.dense_cell_feature_layers is not None and base_type == 'cell':
dense_feature_layers = args.dense_cell_feature_layers
elif args.dense_drug_feature_layers is not None and base_type == 'drug':
dense_feature_layers = args.dense_drug_feature_layers
else:
dense_feature_layers = args.dense_feature_layers
input_model = build_feature_model(
input_shape=shape,
name=fea_type,
dense_layers=dense_feature_layers,
dropout_rate=dropout_rate,
permanent_dropout=permanent_dropout)
encoded = input_model(fea_input)
else:
encoded = fea_input
encoded_inputs.append(encoded)
merged = Concatenate(axis=1)(encoded_inputs)
h = merged
for i, layer in enumerate(args.dense):
x = h
h = Dense(layer, activation=args.activation)(h)
if dropout_rate > 0:
if permanent_dropout:
h = PermanentDropout(dropout_rate)(h)
else:
h = Dropout(dropout_rate)(h)
if args.residual:
try:
h = Add([h, x])
except ValueError:
pass
output = Dense(1)(h)
return Model(inputs, output)
def cnn_concat():
num_classes = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
input_tensor = Input(batch_shape=[0, 1, 28, 28], dtype="float32")
t1 = Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor)
t2 = Conv2D(filters=32,
input_shape=(1, 28, 28),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(input_tensor)
output = Concatenate(axis=1)([t1, t2])
output = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")(output)
output = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
padding="valid")(output)
output = Flatten()(output)
output = Dense(128, activation="relu")(output)
output = Dense(num_classes)(output)
output = Activation("softmax")(output)
model = Model(input_tensor, output)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit(x_train, y_train, epochs=1)
def mlp_concat():
num_classes = 10
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
y_train = np.reshape(y_train, (len(y_train), 1))
#y_train = np.random.randint(1, 9, size=(len(y_train),1), dtype='int32')
print("shape: ", x_train.shape)
input_tensor = Input(batch_shape=[0, 784], dtype="float32")
t1 = Dense(512, input_shape=(784, ), activation="relu",
name="dense1")(input_tensor)
t2 = Dense(512, input_shape=(784, ), activation="relu",
name="dense2")(input_tensor)
t3 = Dense(512, input_shape=(784, ), activation="relu",
name="dense3")(input_tensor)
t4 = Dense(512, input_shape=(784, ), activation="relu",
name="dense4")(input_tensor)
output = Concatenate(axis=1)([t1, t2, t3, t4])
output2 = Dense(512, activation="relu")(output)
output3 = Dense(num_classes)(output2)
output4 = Activation("softmax")(output3)
model = Model(input_tensor, output4)
print(model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
model.compile(optimizer=opt)
model.fit(x_train, y_train, epochs=1)
def top_level_task():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
#teacher
input_tensor1 = Input(shape=(3, 32, 32), dtype="float32")
c1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
c2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
d1 = Dense(512, activation="relu")
d2 = Dense(num_classes)
output_tensor = c1(input_tensor1)
output_tensor = c2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="same")(output_tensor)
output_tensor = c3(output_tensor)
output_tensor = c4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = d1(output_tensor)
output_tensor = d2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
teacher_model = Model(input_tensor1, output_tensor)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
teacher_model.compile(
optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
teacher_model.fit(x_train, y_train, epochs=10)
c1_kernel, c1_bias = c1.get_weights(teacher_model.ffmodel)
c2_kernel, c2_bias = c2.get_weights(teacher_model.ffmodel)
c3_kernel, c3_bias = c3.get_weights(teacher_model.ffmodel)
c4_kernel, c4_bias = c4.get_weights(teacher_model.ffmodel)
d1_kernel, d1_bias = d1.get_weights(teacher_model.ffmodel)
d2_kernel, d2_bias = d2.get_weights(teacher_model.ffmodel)
#d2_kernel *= 0
c2_kernel_new = np.concatenate((c2_kernel, c2_kernel), axis=1)
print(c2_kernel.shape, c2_kernel_new.shape, c2_bias.shape)
#student model
input_tensor2 = Input(shape=(3, 32, 32), dtype="float32")
sc1_1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
sc1_2 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu")
sc2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sd1 = Dense(512, activation="relu")
sd2 = Dense(num_classes)
t1 = sc1_1(input_tensor2)
t2 = sc1_2(input_tensor2)
output_tensor = Concatenate(axis=1)([t1, t2])
output_tensor = sc2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="same")(output_tensor)
output_tensor = sc3(output_tensor)
output_tensor = sc4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = sd1(output_tensor)
output_tensor = sd2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
student_model = Model(input_tensor2, output_tensor)
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
student_model.compile(
optimizer=opt,
loss='sparse_categorical_crossentropy',
metrics=['accuracy', 'sparse_categorical_crossentropy'])
sc1_1.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc1_2.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc2.set_weights(student_model.ffmodel, c2_kernel_new, c2_bias)
sc3.set_weights(student_model.ffmodel, c3_kernel, c3_bias)
sc4.set_weights(student_model.ffmodel, c4_kernel, c4_bias)
sd1.set_weights(student_model.ffmodel, d1_kernel, d1_bias)
sd2.set_weights(student_model.ffmodel, d2_kernel, d2_bias)
student_model.fit(x_train,
y_train,
epochs=160,
callbacks=[
VerifyMetrics(ModelAccuracy.CIFAR10_CNN),
EpochVerifyMetrics(ModelAccuracy.CIFAR10_CNN)
])
def cifar_cnn_net2net():
num_classes = 10
num_samples = 10000
(x_train, y_train), (x_test, y_test) = cifar10.load_data(num_samples)
x_train = x_train.astype('float32')
x_train /= 255
#x_train *= 0
#y_train = np.random.randint(1, 9, size=(num_samples,1), dtype='int32')
y_train = y_train.astype('int32')
print("shape: ", x_train.shape)
#teacher
input_tensor1 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
c1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
c4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
d1 = Dense(512, activation="relu")
d2 = Dense(num_classes)
output_tensor = c1(input_tensor1)
output_tensor = c2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = c3(output_tensor)
output_tensor = c4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = d1(output_tensor)
output_tensor = d2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
teacher_model = Model(input_tensor1, output_tensor)
print(teacher_model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
teacher_model.compile(optimizer=opt)
teacher_model.fit(x_train, y_train, epochs=1)
c1_kernel, c1_bias = c1.get_weights(teacher_model.ffmodel)
c2_kernel, c2_bias = c2.get_weights(teacher_model.ffmodel)
c3_kernel, c3_bias = c3.get_weights(teacher_model.ffmodel)
c4_kernel, c4_bias = c4.get_weights(teacher_model.ffmodel)
d1_kernel, d1_bias = d1.get_weights(teacher_model.ffmodel)
d2_kernel, d2_bias = d2.get_weights(teacher_model.ffmodel)
#d2_kernel *= 0
c2_kernel_new = np.concatenate((c2_kernel, c2_kernel), axis=1)
print(c2_kernel.shape, c2_kernel_new.shape, c2_bias.shape)
#student model
input_tensor2 = Input(batch_shape=[0, 3, 32, 32], dtype="float32")
sc1_1 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc1_2 = Conv2D(filters=32,
input_shape=(3, 32, 32),
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc2 = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc3 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sc4 = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
activation="relu")
sd1 = Dense(512, activation="relu")
sd2 = Dense(num_classes)
t1 = sc1_1(input_tensor2)
t2 = sc1_2(input_tensor2)
output_tensor = Concatenate(axis=1)([t1, t2])
output_tensor = sc2(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = sc3(output_tensor)
output_tensor = sc4(output_tensor)
output_tensor = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
padding="valid")(output_tensor)
output_tensor = Flatten()(output_tensor)
output_tensor = sd1(output_tensor)
output_tensor = sd2(output_tensor)
output_tensor = Activation("softmax")(output_tensor)
student_model = Model(input_tensor2, output_tensor)
print(student_model.summary())
opt = flexflow.keras.optimizers.SGD(learning_rate=0.01)
student_model.compile(optimizer=opt)
sc1_1.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc1_2.set_weights(student_model.ffmodel, c1_kernel, c1_bias)
sc2.set_weights(student_model.ffmodel, c2_kernel_new, c2_bias)
sc3.set_weights(student_model.ffmodel, c3_kernel, c3_bias)
sc4.set_weights(student_model.ffmodel, c4_kernel, c4_bias)
sd1.set_weights(student_model.ffmodel, d1_kernel, d1_bias)
sd2.set_weights(student_model.ffmodel, d2_kernel, d2_bias)
student_model.fit(x_train, y_train, epochs=1)