def test_get_from_module_uses_custom_object():
get_custom_objects().clear()
assert (get_from_module("CustomObject", globals(), "test_generic_utils") == CustomObject)
with pytest.raises(ValueError):
get_from_module("TestObject", globals(), "test_generic_utils")
with custom_object_scope({"TestObject": CustomObject}):
assert (get_from_module("TestObject", globals(), "test_generic_utils") == CustomObject)
'''
def __init__(self, activation, **kwargs):
super(Mish, self).__init__(activation, **kwargs)
self.__name__ = 'Mish'
def mish(x, fast=False):
if fast: # faster but requires extra storage
y = K.exp(-x)
z = 1 + 2 * y
return x * z / (z + 2 * y * y)
#return x * tf.math.tanh(tf.math.softplus(x))
return x * K.tanh(K.softplus(x))
get_custom_objects().update({'Mish': Mish(mish)})
def space_to_depth_x2(x):
return tf.space_to_depth(x, block_size=2)
def FullYOLO(
X, Y
): # after https://github.com/experiencor/basic-yolo-keras/blob/master/backend.py
# see Table 6 in YOLO9000 paper, https://arxiv.org/pdf/1612.08242.pdf
input_image = Input(shape=X[0].shape)
# the function to implement the orgnization layer (thanks to github.com/allanzelener/YAD2K)
def space_to_depth_x2(x):
return tf.space_to_depth(x, block_size=2)
def compute_output_shape(self, input_shape):
factor = self.upsampling_factor * self.upsampling_factor
input_shape_1 = None
if input_shape[1] is not None:
input_shape_1 = input_shape[1] * self.upsampling_factor
input_shape_2 = None
if input_shape[2] is not None:
input_shape_2 = input_shape[2] * self.upsampling_factor
dims = [
input_shape[0], input_shape_1, input_shape_2,
int(input_shape[3] / factor)
]
return tuple(dims)
get_custom_objects().update({'SubpixelConv2D': SubpixelConv2D})
'''
Usage:
x = SubpixelConv2D(upsampling_factor=4)(ip)
'''
#%%
class ICNR:
"""ICNR initializer for checkerboard artifact free sub pixel convolution
Ref:
[1] Andrew Aitken et al. Checkerboard artifact free sub-pixel convolution
https://arxiv.org/pdf/1707.02937.pdf)
Copy:
https://github.com/kostyaev/ICNR
Args:
# In[362]:
# custom activation from Bagnall 2015
# does not appear to perform as well as a ReLU
import tensorflow as tf
from keras.utils.generic_utils import get_custom_objects
def ReSQRT(x):
cond = tf.less_equal(x, 0.0)
result = tf.where(cond, x * 0.0, tf.sqrt(x + 1) - 1)
return result
get_custom_objects().update({'ReSQRT': ReSQRT})
# Bagnall proposes that the following possible values contribute to the success of the model:
#
# | meta-parameter | typical value |
# |--------------------------------- |------------------------------------ |
# | initial adagrad learning scale | 0.1, 0.14, 0.2, 0.3 |
# | initial leakage between classes | 1/4N to 5/N |
# | leakage decay (per sub-epoch) | 0.67 to 0.9 |
# | hidden neurons | 79, 99, 119, 139 |
# | presynaptic noise σ | 0, 0.1, 0.2, 0.3, 0.5 |
# | sub-epochs | 6 to 36 |
# | text direction | forward or backward |
# | text handling | sequential, concatenated, balanced |
# | initialisation | gaussian, zero |
def set_swish():
get_custom_objects().update({"swish": layers.Activation(swish)})
# broadcast_beta = K.reshape(self.beta, (1, G, C//G, 1, 1))
# outputs = outputs + broadcast_beta
# outputs = K.reshape(outputs, (-1, C, H, W))
# outputs = K.permute_dimensions(outputs, (0, 2, 3, 1))
# return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer':
initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer':
regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
}
base_config = super(GroupNormalization, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
get_custom_objects().update({'GroupNormalization': GroupNormalization})
steps_per_epoch = int(num_frame / batch_size)
continue_training = False
saved_model = 'saved_model/fv_model_for_car_June_28_99.h5'
if not continue_training:
print('Initiate training')
model = fcn_model(summary=False)
opt = Adam(lr=1e-4)
#keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=opt, loss=my_loss)
else:
print('Continue training')
from keras.utils.generic_utils import get_custom_objects
get_custom_objects().update({"my_loss": my_loss})
model = load_model(saved_model)
opt = Adam(lr=1e-5)
# #keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(optimizer=opt, loss=my_loss)
checkpointer = ModelCheckpoint(
'saved_model/fv_model_for_car_June_28_{epoch:02d}.h5')
#logger = CSVLogger(filename='saved_model/model_May_29_450.csv')
print(
'Start training - batch_size : {0} - num_frame : {1} - steps_per_epoch : {2}'
.format(batch_size, num_frame, steps_per_epoch))
start = time.time()
broadcast_shape = [1] * len(input_shape)
if self.axis is not None:
broadcast_shape[self.axis] = input_shape[self.axis]
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
normed = normed * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
normed = normed + broadcast_beta
return normed
def get_config(self):
config = {
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
}
base_config = super(InstanceNormalization, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'InstanceNormalization': InstanceNormalization})
import os
import keras.backend as K
from keras.utils.generic_utils import get_custom_objects
def Swish(x):
return (K.sigmoid(x) * x)
class Swish_Class(Activation):
def __init__(self, activation, **kwargs):
super(Swish_Class, self).__init__(activation, **kwargs)
self.__name__ = 'SWISH'
get_custom_objects().update({'Swish': Swish_Class(Swish)})
# Training parameters
batch_size = 128 # orig paper trained all networks with batch_size=128
epochs = 200
data_augmentation = True
num_classes = 10
# Subtracting pixel mean improves accuracy
subtract_pixel_mean = True
# Model parameter
# ----------------------------------------------------------------------------
# | | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch
# Model | n | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti
# |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)
padding_height = max(kernel_height - self.stride, 0)
else:
padding_height = max(kernel_height - (in_height % self.stride), 0)
if (in_width % self.stride == 0):
padding_width = max(kernel_width - self.stride, 0)
else:
padding_width = max(kernel_width- (in_width % self.stride), 0)
padding_top = padding_height // 2
padding_bot = padding_height - padding_top
padding_left = padding_width // 2
padding_right = padding_width - padding_left
return pad(x, [[0,0],
[padding_top, padding_bot],
[padding_left, padding_right],
[0,0] ],
'REFLECT')
def get_config(self):
config = {'stride': self.stride,
'kernel_size': self.kernel_size}
base_config = super(ReflectionPadding2D, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
# Update layers into Keras custom objects
for name, obj in inspect.getmembers(sys.modules[__name__]):
if inspect.isclass(obj) and obj.__module__ == __name__:
get_custom_objects().update({name: obj})
if __name__ != "__main__":
from ..util import isstring, istuple, islist
else:
from nifty.util import isstring, istuple, islist
#####################################################################################################################################################
#####
##### LAYERS & ACTIVATIONS
#####
def Relu(x): return Activation('relu')(x)
def LeakyReLU(x): return keras.layers.LeakyReLU()(x)
def Softmax(x): return Activation('softmax')(x)
get_custom_objects().update({'Relu': Relu})
# get_custom_objects().update({'Leaky_relu': LeakyReLU})
def relu_BN(y):
"Relu activation preceeded by BatchNormalization."
y = BatchNormalization()(y)
y = Relu(y)
return y
def leaky_BN(y):
"LeakyReLU activation preceeded by BatchNormalization."
y = BatchNormalization()(y)
y = keras.layers.LeakyReLU()(y)
return y
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from keras.layers import Activation
from keras.utils.generic_utils import get_custom_objects
def mod(x):
return (x%2)
get_custom_objects().update({'mod': Activation(mod)})
# fix random seed for reproducibility
seed = 7
np.random.seed(seed)
# Dataset preparation
X = np.arange(1, 100)
# X = np.arange(1, 10)
np.random.shuffle(X)
# We now have an array with element n , 1 <= n < 100000
# Our output variable will have the string either ODD or EVEN.
# We will use the numpy vectorize vectorize function to generate
# output dataset.
from keras.layers import Dropout, Activation, Input
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import GlobalAveragePooling2D
from keras.regularizers import l2
from keras.models import Model
from keras import backend as K
from keras.utils.generic_utils import get_custom_objects
def relu_neg(x):
return K.relu(x + 1) - 1
get_custom_objects().update({'relu_neg': Activation(relu_neg)})
def create_model(nb_classes, input_shape, config=None):
"""Create a VGG-16 like model."""
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, "
"nb_cols) or (nb_rows, nb_cols, nb_channels), "
"depending on your backend.")
if config is None:
config = {'model': {}}
min_feature_map_dimension = min(input_shape[:2])
if min_feature_map_dimension < 32:
print("ERROR: Please upsample the feature maps to have at least "
"a size of 32 x 32. Currently, it has {}".format(input_shape))
def test_custom_object_scope_adds_objects():
get_custom_objects().clear()
assert (len(get_custom_objects()) == 0)
with custom_object_scope({"Test1": object, "Test2": object}, {"Test3": object}):
assert (len(get_custom_objects()) == 3)
assert (len(get_custom_objects()) == 0)
#####
def Relu(x):
return Activation('relu')(x)
def LeakyReLU(x):
return keras.layers.LeakyReLU()(x)
def Softmax(x):
return Activation('softmax')(x)
get_custom_objects().update({'Relu': Relu})
# get_custom_objects().update({'Leaky_relu': LeakyReLU})
def relu_BN(y):
"Relu activation preceeded by BatchNormalization."
y = BatchNormalization()(y)
y = Relu(y)
return y
def leaky_BN(y):
"LeakyReLU activation preceeded by BatchNormalization."
y = BatchNormalization()(y)
y = keras.layers.LeakyReLU()(y)
return y
#h._uses_learning_phase = True
c._uses_learning_phase = True
return c, [c, c, t]
#return h, [h, c, t]
def get_config(self):
config = {'units': self.units,
'retention_ratio': self._retention_ratio,
'learn_retention_ratio': self._learn_retention_ratio,
'activation': activations.serialize(self.activation),
'recurrent_activation': activations.serialize(self.recurrent_activation),
'use_bias': self.use_bias,
'kernel_initializer': initializers.serialize(self.kernel_initializer),
'recurrent_initializer': initializers.serialize(self.recurrent_initializer),
'bias_initializer': initializers.serialize(self.bias_initializer),
'unit_forget_bias': self.unit_forget_bias,
'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),
'recurrent_regularizer': regularizers.serialize(self.recurrent_regularizer),
'bias_regularizer': regularizers.serialize(self.bias_regularizer),
'activity_regularizer': regularizers.serialize(self.activity_regularizer),
'kernel_constraint': constraints.serialize(self.kernel_constraint),
'recurrent_constraint': constraints.serialize(self.recurrent_constraint),
'bias_constraint': constraints.serialize(self.bias_constraint),
'dropout': self.dropout,
'recurrent_dropout': self.recurrent_dropout}
base_config = super(PlpRNN, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'PlpRNN': PlpRNN})
def __init__(self):
def plotloss():
plt.figure()
ax = plt.gca()
y1 = R_variable['loss_test']
y2 = R_variable['loss_train']
plt.plot(y1, 'ro', label='Test')
plt.plot(y2, 'g*', label='Train')
# ax.set_xscale('log')
ax.set_yscale('log')
plt.legend(fontsize=18)
plt.xlabel('Epoch', fontsize=15)
plt.title('loss', fontsize=15)
fntmp = '%sloss' % (self.FolderName)
mySaveFig(plt, fntmp, ax=ax, isax=1, iseps=0)
def plotacc():
plt.figure()
ax = plt.gca()
y1 = R_variable['acc_test']
y2 = R_variable['acc_train']
plt.plot(y1, 'ro', label='Test')
plt.plot(y2, 'g*', label='Train')
# ax.set_xscale('log')
# ax.set_yscale('log')
plt.legend(fontsize=18)
plt.xlabel('Epoch', fontsize=15)
plt.title('accuracy', fontsize=15)
fntmp = '%saccuracy' % (self.FolderName)
mySaveFig(plt, fntmp, ax=ax, isax=1, iseps=0)
# 保存文件
def savefile():
# 序列化变量R, 需要的话可以load出来
with open('%s/objs.pkl' % (self.FolderName),
'wb') as f: # Python 3: open(..., 'wb')
pickle.dump(R_variable, f, protocol=4)
# 保存变量R参数长度小于等于20的
text_file = open("%s/Output.txt" % (self.FolderName), "w")
for para in R_variable:
if np.size(R_variable[para]) > 20:
continue
text_file.write('%s: %s\n' % (para, R_variable[para]))
text_file.close()
# 保存loss到csv中
da = pd.DataFrame(R_variable['loss_train'])
da.to_csv(self.FolderName + "loss_train" + ".csv",
header=False,
columns=None)
db = pd.DataFrame(R_variable['loss_test'])
db.to_csv(self.FolderName + "loss_test" + ".csv",
header=False,
columns=None)
dc = pd.DataFrame(R_variable['acc_train'])
dc.to_csv(self.FolderName + "acc_train" + ".csv",
header=False,
columns=None)
dd = pd.DataFrame(R_variable['acc_test'])
dd.to_csv(self.FolderName + "acc_test" + ".csv",
header=False,
columns=None)
# 记录误差值，L2，在每次画loss前更新(以防中期停止程序)
def gapReocord():
R_variable['final_train_loss'] = R_variable['loss_train'][-1]
R_variable['final_test_loss'] = R_variable['loss_test'][-1]
R_variable['final_train_acc'] = R_variable['acc_train'][-1]
R_variable['final_test_acc'] = R_variable['acc_test'][-1]
# 储存误差
R_variable['loss_test'] = []
R_variable['loss_train'] = []
R_variable['acc_test'] = []
R_variable['acc_train'] = []
# s-relu
def custom_activation(x):
return K.relu(-(x - 1)) * K.relu(x)
get_custom_objects().update({'srelu': Activation(custom_activation)})
# 记时，创建新目录
self.t0 = time.time()
self.FolderName = mk_newfolder()
def Resnet18_block(X, filters, s, stage, block):
"""
Implementation of the identity block as defined in Figure 4
Arguments:
X -- input tensor of shape (m, n_H_prev, n_W_prev, n_C_prev)
f -- integer, specifying the shape of the middle CONV's window for the main path
filters -- python list of integers, defining the number of filters in the CONV layers of the main path
stage -- integer, used to name the layers, depending on their position in the network
block -- string/character, used to name the layers, depending on their position in the network
Returns:
X -- output of the identity block, tensor of shape (n_H, n_W, n_C)
"""
# defining name basis
conv_name_base = 'res' + str(stage) + block + '_branch'
bn_name_base = 'bn' + str(stage) + block + '_branch'
# Retrieve Filters
F1, F2 = filters
# Save the input value. You'll need this later to add back to the main path.
X_shortcut = X
if s != 1:
X_shortcut = Conv2D(filters=F1,
kernel_size=(1, 1),
strides=(s, s),
padding='same',
name=conv_name_base + '2c')(X_shortcut)
X_shortcut = BatchNormalization(axis=3,
name=bn_name_base +
'2c')(X_shortcut)
X_shortcut = Activation(R_variable['actfun'])(X_shortcut)
# First component of main path
X = Conv2D(filters=F1,
kernel_size=(3, 3),
strides=(s, s),
padding='same',
name=conv_name_base + '2a')(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2a')(X)
X = Activation(R_variable['actfun'])(X)
### START CODE HERE ###
# Second component of main path (≈3 lines)
X = Conv2D(filters=F2,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
name=conv_name_base + '2b')(X)
X = BatchNormalization(axis=3, name=bn_name_base + '2b')(X)
X = Activation(R_variable['actfun'])(X)
# Final step: Add shortcut value to main path, and pass it through a RELU activation (≈2 lines)
X = layers.add([X, X_shortcut])
X = Activation(R_variable['actfun'])(X)
### END CODE HERE ###
return X
def ResNet18(input_shape=(28, 28, 1), classes=10):
X_input = Input(input_shape)
#X1 = ZeroPadding2D((3,3))(X_input)
#stage 1
X1 = Conv2D(64, (3, 3),
strides=(1, 1),
name='conv1',
padding='same',
kernel_initializer=glorot_uniform(seed=0))(X_input)
X3 = BatchNormalization(axis=3, name='bn_conv1')(X1)
X2 = Activation(R_variable['actfun'])(X3)
X32 = Dropout(0.1)(X2)
#stage2
X3 = Resnet18_block(X32, filters=[64, 64], s=1, stage=2, block='a')
X4 = Resnet18_block(X3, filters=[64, 64], s=1, stage=2, block='b')
X33 = Dropout(0.1)(X4)
#Stage3
X5 = Resnet18_block(X33,
filters=[128, 128],
s=2,
stage=3,
block='a')
X6 = Resnet18_block(X5,
filters=[128, 128],
s=1,
stage=3,
block='b')
X34 = Dropout(0.1)(X6)
#Stage4
X7 = Resnet18_block(X34,
filters=[256, 256],
s=2,
stage=4,
block='a')
X8 = Resnet18_block(X7,
filters=[256, 256],
s=1,
stage=4,
block='b')
X35 = Dropout(0.1)(X8)
#Stage5
X9 = Resnet18_block(X35,
filters=[1024, 1024],
s=2,
stage=5,
block='a')
X10 = Resnet18_block(X9,
filters=[1024, 1024],
s=1,
stage=5,
block='b')
X19 = Resnet18_block(X10,
filters=[1024, 1024],
s=1,
stage=5,
block='c')
X11 = AveragePooling2D(pool_size=(4, 4))(X19)
X12 = Dropout(0.1)(X11)
X23 = Flatten()(X12)
X24 = Dense(1024,
activation=R_variable['actfun'],
kernel_initializer=glorot_uniform(seed=1),
name='dense1')(X23)
X25 = Dense(1024,
activation=R_variable['actfun'],
kernel_initializer=glorot_uniform(seed=2),
name='dense2')(X24)
X26 = Dense(classes,
activation='softmax',
name='fc' + str(classes),
kernel_initializer=glorot_uniform(seed=3))(X25)
model = Model(inputs=X_input, outputs=X26, name='ResNet18')
return model
self.model = ResNet18(input_shape=(28, 28, 1), classes=10)
self.sgd = tf.keras.optimizers.SGD(learning_rate=0.01,
momentum=0.9,
decay=1e-6,
nesterov=True,
name='SGD')
self.adam1 = tf.keras.optimizers.Adam(learning_rate=0.01,
beta_1=0.85,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam1')
self.adam2 = tf.keras.optimizers.Adam(learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam2')
self.adam3 = tf.keras.optimizers.Adam(learning_rate=0.0005,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam3')
self.adam4 = tf.keras.optimizers.Adam(learning_rate=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam4')
self.adam5 = tf.keras.optimizers.Adam(learning_rate=0.00001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07,
amsgrad=False,
name='Adam4')
def fn(correct, predicted):
return tf.nn.softmax_cross_entropy_with_logits(labels=correct,
logits=predicted)
self.model.compile(optimizer=self.adam2,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
early_stopping = EarlyStopping(monitor='val_loss',
patience=30,
verbose=2)
self.history1 = self.model.fit(
R_variable['train_inputs'],
R_variable['y_true_train'],
epochs=50,
batch_size=256,
validation_data=(R_variable['test_inputs'],
R_variable['y_true_test']),
shuffle=True)
R_variable['loss_train'] = self.history1.history['loss']
R_variable['loss_test'] = self.history1.history['val_loss']
R_variable['acc_train'] = self.history1.history['accuracy']
R_variable['acc_test'] = self.history1.history['val_accuracy']
round_time = time.time()
R_variable['use_time'] = round_time - self.t0
savefile()
gapReocord()
plotloss()
plotacc()
self.model.compile(optimizer=self.adam4,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
self.history2 = self.model.fit(
R_variable['train_inputs'],
R_variable['y_true_train'],
epochs=25,
batch_size=256,
validation_data=(R_variable['test_inputs'],
R_variable['y_true_test']),
shuffle=True,
callbacks=[early_stopping])
R_variable['loss_train'] += self.history2.history['loss']
R_variable['loss_test'] += self.history2.history['val_loss']
R_variable['acc_train'] += self.history2.history['accuracy']
R_variable['acc_test'] += self.history2.history['val_accuracy']
round_time = time.time()
R_variable['use_time'] = round_time - self.t0
savefile()
gapReocord()
plotloss()
plotacc()
self.model.compile(optimizer=self.adam5,
loss=tf.keras.losses.CategoricalCrossentropy(),
metrics=['accuracy'])
self.history3 = self.model.fit(
R_variable['train_inputs'],
R_variable['y_true_train'],
epochs=25,
batch_size=256,
validation_data=(R_variable['test_inputs'],
R_variable['y_true_test']),
shuffle=True,
callbacks=[early_stopping])
R_variable['loss_train'] += self.history3.history['loss']
R_variable['loss_test'] += self.history3.history['val_loss']
R_variable['acc_train'] += self.history3.history['accuracy']
R_variable['acc_test'] += self.history3.history['val_accuracy']
round_time = time.time()
R_variable['use_time'] = round_time - self.t0
savefile()
gapReocord()
plotloss()
plotacc()
score1 = self.model.evaluate(R_variable['train_inputs'],
R_variable['y_true_train'],
verbose=0)
score2 = self.model.evaluate(R_variable['test_inputs'],
R_variable['y_true_test'],
verbose=0)
print("Program ends. ")
print("Train accuracy is: %3.3f, Train loss is: %3.3f" %
(score1[1], score1[0]))
print("Test accuracy is: %3.3f, Test loss is: %3.3f" %
(score2[1], score2[0]))
print("The program have been running for %ds." %
(time.time() - self.t0))
dim = K.int_shape(mean)[1]
latent_loss = -0.5 * (1 + log_var - K.square(mean) - K.exp(log_var))
latent_loss = K.sum(latent_loss, axis=1, keepdims=True)
latent_loss = K.mean(latent_loss)
latent_loss = self.gamma * K.abs(latent_loss - self.max_capacity)
latent_loss = K.reshape(latent_loss, [1, 1])
epsilon = K.random_normal(shape=(batch, dim), mean=0., stddev=1.)
layer_output = mean + K.exp(0.5 * log_var) * epsilon
self.add_loss(losses=[latent_loss], inputs=[layer_inputs])
return layer_output
def compute_output_shape(self, input_shape):
return input_shape[0]
def get_config(self):
config = {
'gamma': self.gamma,
'capacity': self.max_capacity,
'name': self.name
}
base_config = super(SampleLayer, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'SampleLayer': SampleLayer})
def build(self, input_shape):
pass
def call(self, x, mask=None):
y = tf.compat.v1.depth_to_space(input=x,
block_size=self.scale_factor,
name=self.data_format)
return y
def compute_output_shape(self, input_shape):
if self.data_format == 'channels_first':
b, k, r, c = input_shape
return (b, k // (self.scale_factor**2), r * self.scale_factor,
c * self.scale_factor)
else:
b, r, c, k = input_shape
return (b, r * self.scale_factor, c * self.scale_factor,
k // (self.scale_factor**2))
def get_config(self):
config = {
'scale_factor': self.scale_factor,
'data_format': self.data_format
}
base_config = super(SubPixelUpscaling, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'SubPixelUpscaling': SubPixelUpscaling})
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
#custom activation
from keras.layers import Activation
from keras.utils.generic_utils import get_custom_objects
def srelu(x):
if np.random.rand(1) > 0.5:
return K.relu(x)
return K.relu(x) - x
get_custom_objects().update({'srelu': Activation(srelu)})
def baseline_model(af):
# create model
model = Sequential()
model.add(Conv2D(32, (5, 5), input_shape=(1, 28, 28), activation=af))
model.add(Flatten())
model.add(Dense(128, activation=af))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
#validation_data=validation_generator.get_next_batch(),
#validation_steps=nb_validation_samples // batch_size,
# max_queue_size=5
#callbacks=[history, tb, earlystop],
class_weight=class_weights)
model.save('my_model_inception_2_full.h5')
# save history
try:
with open('history_dict_earlystop_full.pkl', 'wb') as f:
pickle.dump(history.history, f)
except:
print("history object does not exist.")
print("Maybe you did a full training without validation.")
# load model (need to add the "custom" metric function again)
top20_acc = functools.partial(top_k_categorical_accuracy, k=20)
top20_acc.__name__ = 'top20_acc'
keras.losses.custom_loss = top20_acc
get_custom_objects().update({"top20_acc": top20_acc})
print("check of loading model works...")
model = load_model('my_model.h5')
print(model.summary())
# preds = model.evaluate_generator(validation_generator.get_next_batch(),
# steps=nb_validation_samples\
# // batch_size)
def call(self, inputs, training=None):
def drop_connect():
keep_prob = 1.0 - self.drop_connect_rate
# Compute drop_connect tensor
batch_size = tf.shape(inputs)[0]
random_tensor = keep_prob
random_tensor += K.random_uniform([batch_size, 1, 1, 1],
dtype=inputs.dtype)
binary_tensor = tf.floor(random_tensor)
output = (inputs / keep_prob) * binary_tensor
return output
return K.in_train_phase(drop_connect, inputs, training=training)
def get_config(self):
config = {
'drop_connect_rate': self.drop_connect_rate,
}
base_config = super(DropConnect, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({
'EfficientNetConvInitializer': EfficientNetConvInitializer,
'EfficientNetDenseInitializer': EfficientNetDenseInitializer,
'DropConnect': DropConnect,
'Swish': Swish,
})
import numpy as np
np.random.seed(1337) # for reproducibility
from keras.models import Sequential
from keras.layers import Dense, Activation, Convolution2D, MaxPooling2D, Flatten, LocallyConnected2D, ZeroPadding2D
from keras.optimizers import Adam
from keras.initializers import random_uniform
from six.moves import cPickle as pickle
from keras import backend as K
from keras.utils.generic_utils import get_custom_objects
def scaled_hyperbolic_tangent(x):
return K.tanh((2 / 3) * x) * 1.7159
get_custom_objects().update(
{'scaled_hyperbolic_tangent': Activation(scaled_hyperbolic_tangent)})
with open('./train_data/data.pickle', 'rb') as f:
tr_dat = pickle.load(f)
with open('./train_data/label.pickle', 'rb') as f:
tr_lab = pickle.load(f)
with open('./test_data/data.pickle', 'rb') as f:
te_dat = pickle.load(f)
with open('./test_data/label.pickle', 'rb') as f:
te_lab = pickle.load(f)
# zero_padding (now the data shape is [None, 16, 16])
# after zero_padding -> [None, 18, 18]
tr_dat = np.pad(tr_dat, ((0, 0), (1, 1), (1, 1)), 'constant')
te_dat = np.pad(te_dat, ((0, 0), (1, 1), (1, 1)), 'constant')
from pandas import read_csv
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
from keras.utils.np_utils import to_categorical
from keras.utils.generic_utils import get_custom_objects
import numpy as np
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import os
def normalized_linear(x):
return x / sum(x, axis=-1, keepdims=True)
get_custom_objects().update(
{"normalized_linear": Activation(normalized_linear)})
class SGAN:
def __init__(self):
self.num_classes = 4
self.features = 12
self.timesteps = 40
self.timesteps_in_future = 20
self.nodes_per_layer = 32
self.filter_length = 3
self.generator_input_length = 100
self.dropout = 0.2
self.encoder = LabelEncoder()
self.results_folder_name = ""
self.results_folder_name_gs = ""
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import MaxPooling2D
from keras import optimizers
from sim_layer import Norm_Conv2d as Conv2D
# create custom activation function for CosSim layer
def pow_activation(x, p_val=3):
#return 2 * K.pow(K.tanh(x) , p_val)
return K.pow(x, p_val)
get_custom_objects().update({'pow_activation': Activation(pow_activation)})
def lr_schedule(epoch):
lrate = 0.001
if epoch > 25:
lrate = 0.0005
if epoch > 50:
lrate = 0.0002
return lrate
batch_size = 32
epochs = 75
# The data, split between train and test sets:
return tf.nn.swish(inputs)
class DropConnect(KL.Layer):
def __init__(self, drop_connect_rate=0.0, **kwargs):
super().__init__(**kwargs)
self.drop_connect_rate = drop_connect_rate
def call(self, inputs, training=None):
def drop_connect():
keep_prob = 1.0 - self.drop_connect_rate
# Compute drop_connect tensor
batch_size = tf.shape(inputs)[0]
random_tensor = keep_prob
random_tensor += tf.random_uniform([batch_size, 1, 1, 1],
dtype=inputs.dtype)
binary_tensor = tf.floor(random_tensor)
output = tf.div(inputs, keep_prob) * binary_tensor
return output
return K.in_train_phase(drop_connect, inputs, training=training)
def get_config(self):
config = super().get_config()
config["drop_connect_rate"] = self.drop_connect_rate
return config
get_custom_objects().update({"DropConnect": DropConnect, "Swish": Swish})
if `data_format` is `"channels_last"`.
# Output shape
5D tensor with shape:
`(samples, channels + 2, conv_dim1, conv_dim2, conv_dim3)`
if `data_format` is `"channels_first"`
or 5D tensor with shape:
`(samples, conv_dim1, conv_dim2, conv_dim3, channels + 2)`
if `data_format` is `"channels_last"`.
# References:
- [An Intriguing Failing of Convolutional Neural Networks and the CoordConv Solution](https://arxiv.org/abs/1807.03247)
"""
def __init__(self, data_format=None,
**kwargs):
super(CoordinateChannel3D, self).__init__(
rank=3,
use_radius=False,
data_format=data_format,
**kwargs
)
def get_config(self):
config = super(CoordinateChannel3D, self).get_config()
config.pop('rank')
config.pop('use_radius')
return config
get_custom_objects().update({'CoordinateChannel1D': CoordinateChannel1D,
'CoordinateChannel2D': CoordinateChannel2D,
'CoordinateChannel3D': CoordinateChannel3D})
from keras import layers, models, optimizers, regularizers
from keras import backend as K
from keras.layers import Flatten, Concatenate, LeakyReLU
from keras.utils.generic_utils import get_custom_objects
from keras.initializers import RandomUniform, Zeros
from keras_radam import RAdam
def mish(x):
return x * K.tanh(K.softplus(x))
get_custom_objects().update({'Mish': layers.Activation(mish)})
class Actor:
"""Actor (Policy) Model."""
def __init__(self, state_size, action_size, action_low, action_high, lr,
network):
"""Initialize parameters and build model.
Params
======
state_size (int): Dimension of each state
action_size (int): Dimension of each action
action_low (array): Min value of each action dimension
action_high (array): Max value of each action dimension
"""
self.state_size = state_size
self.action_size = action_size
self.action_low = action_low
import numpy as np
import keras2onnx
import onnxruntime
import tensorflow as tf
from keras import backend as K
from tensorflow.keras.models import load_model
import numpy as np
import sys
from constraints import ZeroSomeWeights
from keras.utils.generic_utils import get_custom_objects
get_custom_objects().update({"ZeroSomeWeights": ZeroSomeWeights})
model = load_model(sys.argv[1])
X = np.array(np.random.rand(10, 21), dtype=np.float32)
print(model.predict(X))
# convert to onnx model
onnx_model = keras2onnx.convert_keras(model, model.name)
temp_model_file = 'NN_model.onnx'
keras2onnx.save_model(onnx_model, temp_model_file)
sess = onnxruntime.InferenceSession(temp_model_file)
input_name = sess.get_inputs()[0].name
label_name = sess.get_outputs()[0].name
np.random.seed(5566)
from keras import backend as K
from keras.layers import Activation, Input, ZeroPadding2D
from keras.layers.core import Dense, Lambda, Reshape
from keras.layers.convolutional import Convolution2D
from keras.layers.pooling import AveragePooling2D, GlobalMaxPooling2D
from keras.layers.merge import concatenate, dot
from keras.models import Model
from keras.utils.generic_utils import get_custom_objects
def log_activation(x):
return K.log(x)
get_custom_objects().update({'log_activation': Activation(log_activation)})
def create_model(MAX_QRY_LENGTH=50,
MAX_DOC_LENGTH=2900,
NUM_OF_FEATS=10,
PSGS_SIZE=[(50, 1)],
NUM_OF_FILTERS=5,
tau=1):
alpha_size = len(PSGS_SIZE)
psgMat = Input(shape=(
MAX_QRY_LENGTH,
MAX_DOC_LENGTH,
1,
), name="passage")
homoMat = Input(shape=(NUM_OF_FEATS, ), name="h_feats")
tf.set_random_seed(1234)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
K.set_session(sess)
#Define activation functions
def tf_sqnlsig(x):
u = tf.clip_by_value(x, -2, 2)
a = u
b = tf.negative(tf.abs(u))
wsq = (tf.multiply(a, b)) / 4.0
y = tf.add(tf.multiply(tf.add(u, wsq), 0.5), 0.5)
return y
get_custom_objects().update({'custom_activation': Activation(tf_sqnlsig)})
def tf_sqnl(x):
u = tf.clip_by_value(x, -2, 2)
a = u
b = tf.negative(tf.abs(u))
wsq = (tf.multiply(a, b)) / 4.0
y = tf.add(u, wsq)
return y
get_custom_objects().update({'custom_activation': Activation(tf_sqnl)})
def tf_tansig(x):
def __init__(self,
input_dim,
n_hidden_units,
n_hidden_layers,
rho=0.6,
nonlinearity='tanh',
init='gau',
bias_sigma=0.0,
weight_sigma=1.25,
input_layer=None,
flip=False,
output_dim=None):
#if input_layer is not None:
# assert input_layer.output_shape[1] == input_dim
self.input_dim = input_dim
self.n_hidden_units = n_hidden_units
self.n_hidden_layers = n_hidden_layers
self.nonlinearity = nonlinearity
self.bias_sigma = bias_sigma
self.weight_sigma = weight_sigma
self.input_layer = input_layer
if output_dim is None:
output_dim = n_hidden_units
self.output_dim = output_dim
model = Sequential()
get_custom_objects().update({'hard_tanh': Activation(hard_tanh)})
get_custom_objects().update({'erf2': Activation(erf2)})
if input_layer is not None:
model.add(input_layer)
# model.add(Dropout(0.1, input_layer))
#model.add(Activation('tanh'))
for i in range(n_hidden_layers):
nunits = n_hidden_units if i < n_hidden_layers - 1 else output_dim
if flip:
if nonlinearity == 'prelu':
model.add(
LeakyReLU(alpha=0.5,
input_shape=(input_dim, ),
name='a%d' % i))
else:
model.add(
Activation(nonlinearity,
input_shape=(input_dim, ),
name='a%d' % i))
# model.add(Activation(nonlinearity, input_dim=1000, name='a%d'%i))
model.add(Dropout(1 - rho)) # dropout = 1 - rho
if init == 'gau':
model.add(
Dense(nunits,
name='d%d' % i,
kernel_initializer=normal(
mean=0.0,
stddev=weight_sigma * 1.0 /
np.sqrt(n_hidden_units)),
bias_initializer=normal(mean=0.0,
stddev=bias_sigma)))
if init == 'orth':
model.add(
Dense(nunits,
name='d%d' % i,
kernel_initializer=orthogonal(gain=weight_sigma),
bias_initializer=normal(mean=0.0,
stddev=bias_sigma)))
# model.add(Dense(nunits, name='d%d'%i))
else:
model.add(
Dense(nunits, input_shape=(input_dim, ), name='%d' % i))
if i < n_hidden_layers - 1 or self.output_dim == self.n_hidden_units:
model.add(Activation(nonlinearity, name='a%d' % i))
else:
# Theano is optimizing out the nonlinearity if it can which is breaking shit
# Give it something that it won't optimize out.
model.add(
Activation(lambda x: T.minimum(x, 999999.999),
name='a%d' % i))
model.build()
self.model = model
# print(self.hs)
self.weights = model.get_weights()
self.dense_layers = filter(lambda x: x.name.startswith('d'),
model.layers)
self.activ_layers = filter(lambda x: x.name.startswith('a'),
model.layers)
self.hs = [h.output for h in self.dense_layers]
self.ac = [b.output for b in self.activ_layers]
self.f_acts = self.f_jac = self.f_jac_hess = self.f_act = None
vec = K.ones_like(self.model.input)
embedding_layer = Embedding(len(word_index),
output_dim=emb_size,
weights=[embedding_matrix],
input_length=max_length,
trainable=False)
print("embedd matrix shape: ", embedding_matrix.shape)
batch_size = 64
def swish(x):
return (K.sigmoid(x) * x)
get_custom_objects().update({'swish': Activation(swish)})
for i in range(20):
if i == 0:
numb_train = 180000
train_X, valid_X = train_X_num[:numb_train], train_X_num[numb_train:]
train_y, valid_y = train_y_cat[:numb_train], train_y_cat[numb_train:]
elif i == 1:
numb_train = -180000
train_X, valid_X = train_X_num[numb_train:], train_X_num[:numb_train]
train_y, valid_y = train_y_cat[numb_train:], train_y_cat[:numb_train]
else:
train_X, valid_X, train_y, valid_y = train_test_split(train_X_num,
train_y_cat,
test_size=0.07)
sequence_input = Input(shape=(max_length, ), dtype='int32')
D = np.empty((4, 0))
for i in range(max_iter):
D = np.concatenate([D, generate_timed_trajectory(28)], axis=1)
# format data for training
input_data = D[:3].T
target_data = D[3].T
# ## Set up Neural Network
# set up keras
def rad_bas(x):
return K.exp(-x**2)
get_custom_objects().update({'rad_bas': Activation(rad_bas)})
def tan_sig(x):
return 2 / (1 + K.exp(-2 * x)) - 1
get_custom_objects().update({'tan_sig': Activation(tan_sig)})
# define neural net
model = Sequential()
model.add(Dense(10, activation='tan_sig', use_bias=True, input_shape=(3, )))
model.add(Dense(10, activation='sigmoid', use_bias=True))
model.add(Dense(10, activation='linear', use_bias=True))
model.add(Dense(1))
# Name of the training session to which all objects will be saved
NAME_OF_THE_TRAIN_SESSION = config["NAME_OF_THE_TRAIN_SESSION"]
PATH_TO_THE_LEARNING_SESSION = "./learning_sessions/" + NAME_OF_THE_TRAIN_SESSION + "/"
# Name of the pretrained model, if continue_training=1
pretrained_filepath = PATH_TO_THE_LEARNING_SESSION + config["MODEL_NAME"]
print("DEBUG = {} \nconfig = {}\nmodel_config = {}".format(
DEBUG, config, model_config))
# Update custom object with my own loss functions
custom_objects = {
'macro_averaged_recall_tf_onehot': macro_averaged_recall_tf_onehot,
'macro_averaged_recall_tf_soft': macro_averaged_recall_tf_soft
}
get_custom_objects().update(custom_objects)
# Configuring gpu for the training
gpu_configuration_initialization()
use_generator = 0
# Build directories
path_builder(PATH_TO_THE_LEARNING_SESSION)
# Create logger to files (copying std out stream to a file)
sys.stdout = Logger(PATH_TO_THE_LEARNING_SESSION + "log_training")
# First, build index mapping words in the embeddings set to their embedding vector
print("Indexing word vectors.")
# Using twitter specific dataset. Pretrained by glove.
filename_to_read = config["TWITTER_GLOVE"]
def call(self, inputs):
return resize_images(inputs, self.factor[0], self.factor[1],
self.data_format, self.interpolation)
def get_config(self):
config = {
'factor': self.factor,
'interpolation': self.interpolation,
'data_format': self.data_format
}
base_config = super(ResizeImage, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'ResizeImage': ResizeImage})
def variable(value, dtype=_FLOATX, name=None):
v = tf.Variable(np.asarray(value, dtype=dtype), name=name)
_get_session().run(v.initializer)
return v
def shape(x):
return x.get_shape()
def square(x):
return tf.square(x)
from keras.models import Sequential
from keras.layers import Activation, Dense, Maximum
from keras import backend as K
from keras.utils.generic_utils import get_custom_objects
import keras
import numpy as np
get_custom_objects().update(
{'sine': Activation(lambda x: K.sigmoid(10 * K.sin(x)))})
n_rows = 3
limit = 5
model = Sequential()
model.add(
Dense(units=(limit + 1) * n_rows, activation='sine', input_dim=n_rows))
model.add(Dense(units=limit, activation='sine'))
model.add(Dense(units=1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
npoints_x = 1000000
x_data_r = np.random.randint(0, 2**limit + 1, (npoints_x // 2, n_rows))
y_data_r = np.where(np.bitwise_xor.reduce(x_data_r, axis=1) == 0, 0,
1).reshape((-1, 1))
x_data_0 = np.random.randint(0, 2**limit + 1, (npoints_x // 2, n_rows - 1))
x_data_0 = np.hstack(
(x_data_0, np.bitwise_xor.reduce(x_data_0, axis=1).reshape((-1, 1))))
y_data_0 = np.zeros((npoints_x // 2, 1))
cnn = self.ResidualBlock(64, cnn)
cnn = self.ResidualBlock(64, cnn)
cnn = AveragePooling2D((2, 2))(cnn)
cnn = LeakyReLU(alpha=0.2)(cnn)
cnn = Conv2D(128, (5, 5), padding='same')(cnn)
cnn = LeakyReLU(alpha=0.2)(cnn)
cnn = self.ResidualBlock(128, cnn)
cnn = self.ResidualBlock(128, cnn)
cnn = AveragePooling2D((2, 2))(cnn)
cnn = LeakyReLU(alpha=0.2)(cnn)
cnn = Conv2D(256, (5, 5), padding='same')(cnn)
cnn = LeakyReLU(alpha=0.2)(cnn)
cnn = self.ResidualBlock(256, cnn)
cnn = self.ResidualBlock(256, cnn)
cnn = Reshape((-1, 256))(cnn)
capsule = Capsule(self.class_num, 16, 3, True)(cnn)
output = Lambda(lambda x: K.sqrt(K.sum(K.square(x), 2)),
output_shape=(self.class_num, ))(capsule)
model = Model(inputs=input_image, outputs=output)
return model
get_custom_objects().update({'Capsule': Capsule})
"""Create a sequential model."""
from keras.layers import Dropout, Activation, Input
from keras.layers import Convolution2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import GlobalAveragePooling2D
from keras.regularizers import l2
from keras.models import Model
from keras import backend as K
from keras.utils.generic_utils import get_custom_objects
def sigmoid_neg(x):
return K.sigmoid(x) - 0.5
get_custom_objects().update({'sigmoid_neg': Activation(sigmoid_neg)})
def create_model(nb_classes, input_shape, config=None):
"""Create a VGG-16 like model."""
if len(input_shape) != 3:
raise Exception("Input shape should be a tuple (nb_channels, nb_rows, "
"nb_cols) or (nb_rows, nb_cols, nb_channels), "
"depending on your backend.")
if config is None:
config = {'model': {}}
min_feature_map_dimension = min(input_shape[:2])
if min_feature_map_dimension < 32:
print("ERROR: Please upsample the feature maps to have at least "
"a size of 32 x 32. Currently, it has {}".format(input_shape))
def __init__(self, scale_factor=2, data_format=None, **kwargs):
super(SubPixelUpscaling, self).__init__(**kwargs)
self.scale_factor = scale_factor
self.data_format = normalize_data_format(data_format)
def build(self, input_shape):
pass
def call(self, x, mask=None):
y = tf.depth_to_space(x, self.scale_factor, self.data_format)
return y
def compute_output_shape(self, input_shape):
if self.data_format == 'channels_first':
b, k, r, c = input_shape
return (b, k // (self.scale_factor ** 2), r * self.scale_factor, c * self.scale_factor)
else:
b, r, c, k = input_shape
return (b, r * self.scale_factor, c * self.scale_factor, k // (self.scale_factor ** 2))
def get_config(self):
config = {'scale_factor': self.scale_factor,
'data_format': self.data_format}
base_config = super(SubPixelUpscaling, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
get_custom_objects().update({'SubPixelUpscaling': SubPixelUpscaling})
