def build_model(self, input_shape, nb_classes): input_layer = keras.layers.Input(input_shape) #num_feat=30 nb_filters = 8 kernel_size = 12 dilations = [2**i for i in range(2)] padding = 'causal' nb_stacks = 1 #max_len=X_train[0:1].shape[1] use_skip_connections = True use_batch_norm = True dropout_rate = 0.05 kernel_initializer = 'he_normal' #lr=0.00 activation = 'relu' use_layer_norm = True return_sequences = True #name='tcn_1' enc = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, kernel_initializer, use_batch_norm, use_layer_norm)(input_layer) #output_layer = keras.layers.Dense(nb_classes, emb = keras.layers.Dense(16, activation='relu')(enc) #emb_rep = keras.layers.RepeatVector(input_shape[0])(emb) return_sequences = True nb_filters = 8 decod = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, kernel_initializer, use_batch_norm, use_layer_norm)(emb) output = emb = keras.layers.Dense(input_shape[1], activation='sigmoid')(enc) model = keras.models.Model(inputs=input_layer, outputs=output) model.compile(loss='mse', optimizer=Adam()) self.encoder = keras.models.Model(inputs=input_layer, outputs=enc) self.decoder = keras.models.Model(inputs=input_layer, outputs=decod) self.emb = keras.models.Model(inputs=input_layer, outputs=emb) print(model.summary()) reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=10, min_lr=0.0001) file_path = self.output_directory + 'best_model' model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True) self.callbacks = [reduce_lr, model_checkpoint] return model
def build_discriminator(time_steps, feature_size): print(backend.image_data_format()) inputs = Input(batch_shape=(16, time_steps, feature_size)) output1 = TCN(nb_filters=16, dilations=[1, 2, 4, 8, 16, 32, 64], kernel_size=3, dropout_rate=0.1, return_sequences=True)(inputs) output2 = TCN(nb_filters=32, dilations=[1, 2, 4, 8, 16, 32, 64], kernel_size=3, dropout_rate=0.1, return_sequences=True)(output1) output3 = TCN(nb_filters=32, dilations=[1, 2, 4, 8, 16, 32, 64], kernel_size=3, dropout_rate=0.1, return_sequences=False)(output2) output = Dense(8, activation='softmax')(output3) model = Model(inputs=[inputs], outputs=[output]) model.summary() return model
def define_models(n_input, n_output): # 训练模型中的encoder 32 16 128 encoder_inputs = Input(shape=(None, n_input)) #500 32 encoder = TCN(64, activation='relu', return_sequences=True)(encoder_inputs) encoder2 = TCN(32, activation='relu', return_sequences=True)(encoder) encoder3 = TCN(16, activation='relu')(encoder2) decoder_dense = Dense(n_output, activation='linear')(encoder3) model = Model(encoder_inputs, decoder_dense) return model
def model(self): #build model around TCN #As a general rule (keeping kernel_size fixed at 2) and dilations increasing with a factor of 2 #The equation to find the ideal sizes is receptive field = nb_stacks_of_residuals_blocks(nb_stacks) * kernel_size * last_dilation) #Each layer adds linearly to the receptive field self.built = True i = Input(batch_shape=(self.batch_size, self.moments-1, self.input_dim)) #Model 1: Simple TCN for lower layer and LSTM for upper, set to handel a receptive field of around 64 #for TCN compressed down for LSTM, build for self.moments between 40-80, networth and accuracy can be used for testing ######################################################################### #x1 = TCN(return_sequences=True, nb_filters=32, nb_stacks = 1, dropout_rate=.0, kernel_size=2)(i) #x1 = Dense(4, activation='linear')(x1) #o = LSTM(4, dropout=.3)(x1) ######################################################################### # Model 2: 1*10^-6 error, average networth change per tick = 23/(774-60) = shit, build for self.moments between 40-80, networth and accuracy can be used for testing ######################################################################### #i = LSTM(50, activation='relu', return_sequences=True, input_shape=(n_steps, n_features))(i) #optional addition to try stacked LSTM not added yet #x = LSTM(50, dropout=.3, activation='relu')(i) #o = Dense(4, activation='softmax')(x) ######################################################################### #Model 3: TCN with LSTM on top with dual bottom layer, build for self.moments between 40-80, networth and accuracy can be used for testing ######################################################################### x1 = TCN(return_sequences=True, nb_filters=64, dilations = [1, 2, 4, 8, 16, 32], nb_stacks=1, dropout_rate=.1, kernel_size=2)(i) x2 = Lambda(lambda z: backend.reverse(z, axes=0))(i) x2 = TCN(return_sequences=True, nb_filters=64, dilations = [1, 2, 4, 8, 16, 32], nb_stacks=1, dropout_rate=.1, kernel_size=2)(x2) x = add([x1, x2]) o = LSTM(4, dropout=.1)(x) ######################################################################### # Model 4: Layered TCN with LSTM on top, build for self.moments between 40-80, networth and accuracy can be used for testing ######################################################################### #x1 = TCN(return_sequences=True, nb_filters = 64, dilations = [1, 2, 4, 8, 16, 32], nb_stacks = 1, dropout_rate=.1, kernel_size=2)(i) #x1 = TCN(return_sequences=True, nb_filters = 64, dilations = [1, 2, 4, 8, 16, 32], nb_stacks = 1, dropout_rate=.1, kernel_size=2)(x1) #x1 = Dense(4, activation='linear')(x1) #x2 = LSTM(4, dropout=.3)(i) #x = add([x1, x2]) #o = concatenate([GlobalMaxPooling1D()(x), GlobalAveragePooling1D()(x)]) #o = Dense(4, activation='linear')(o) ######################################################################### # Model 5: Dual TCN layered, build for self.moments between 40-80, networth and accuracy can be used for testing ######################################################################### #x1 = TCN(return_sequences=True, nb_filters=64, dilations =[1, 2, 4, 8, 16, 32], nb_stacks=1, dropout_rate=.1, kernel_size=1)(i) #x2 = Lambda(lambda z: backend.reverse(z, axes=0))(i) #x2 = TCN(return_sequences=True, nb_filters=64, dilations =[1, 2, 4, 8, 16, 32], nb_stacks=1, dropout_rate=.1, kernel_size=1)(x2) #x = add([x1, x2]) #x1 = TCN(return_sequences=True, nb_filters=64, dilations =[1, 2, 4, 8], nb_stacks=1, dropout_rate=.1, kernel_size=1)(x) #x2 = Lambda(lambda z: backend.reverse(z, axes=0))(x) #x2 = TCN(return_sequences=True, nb_filters=64, dilations =[1, 2, 4, 8], nb_stacks=1, dropout_rate=.1, kernel_size=1)(x2) #x = add([x1, x2]) #o = concatenate([GlobalMaxPooling1D()(x), GlobalAveragePooling1D()(x)]) #o = Dense(4, activation='linear')(o) self.m = Model(inputs=i, outputs=o) self.m.compile(optimizer='adam', loss=custom_loss) #optimizer and loss can be changed to what we want
def TCN_model(X_Train, Y_train, batch_size, activation, dilations, nbfilters, kernelsize, nbstacks, batch_norm, layer_norm, weight_norm, dropout_rate, lookback_window): from tcn import TCN, tcn_full_summary from tensorflow.keras.layers import Dense from tensorflow.keras.models import Sequential from tensorflow.keras.callbacks import EarlyStopping from keras.optimizers.schedules import ExponentialDecay from keras.optimizers import Adam import tensorflow as tf from keras.regularizers import l2 batch_size, time_steps, input_dim = batch_size, 10, X_Train.shape[2] lr_schedule = ExponentialDecay(initial_learning_rate=1e-2, decay_steps=1000, decay_rate=0.9) tcn_layer = TCN(input_shape=(lookback_window, X_Train.shape[2]), activation=activation, padding='causal', dilations=dilations, nb_filters=nbfilters, kernel_size=kernelsize, nb_stacks=nbstacks, use_batch_norm=batch_norm, use_layer_norm=layer_norm, use_weight_norm=weight_norm, dropout_rate=dropout_rate) model_all_data = Sequential([ TCN(input_shape=(lookback_window, X_Train.shape[2]), activation=activation, padding='causal', dilations=dilations, nb_filters=nbfilters, kernel_size=kernelsize, nb_stacks=nbstacks, use_batch_norm=batch_norm, use_layer_norm=layer_norm, use_weight_norm=weight_norm, dropout_rate=dropout_rate), Dense( Y_train.shape[1], activation=activation, kernel_regularizer=l2(0.01), ) ]) # The receptive field tells you how far the model can see in terms of timesteps. print('Receptive field size =', tcn_layer.receptive_field) model_all_data.summary() adam = Adam(learning_rate=lr_schedule) model_all_data.compile(optimizer=adam, loss=root_mean_squared_error, metrics=['mse', 'mae', 'mape']) # tcn_full_summary(model_all_data, expand_residual_blocks=False) return model_all_data
def model(self): #build model around TCN self.built = True i = Input(batch_shape=(self.batch_size, self.moments - 1, self.input_dim)) o = TCN(return_sequences=True)(i) o = TCN(return_sequences=False)(o) o = Dense(1)(o) self.m = Model(inputs=i, outputs=o) self.m.compile( optimizer='adam', loss='mse') #optimizer and loss can be changed to what we want
def regression_model(input_dim = 31,tcn = False): if not tcn: from tensorflow.keras import layers, models import tensorflow as tf model = models.Sequential() model.add(layers.GRU(128, return_sequences=True, input_shape=(None, input_dim))) model.add(layers.TimeDistributed(layers.Dense(13, activation='linear'))) model.summary() model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=0.01)) else: import tcn import keras from keras.layers import Dense,TimeDistributed from keras.models import Input, Model from tcn import TCN i = Input(batch_shape=(None, None, input_dim)) o = TCN(return_sequences=True)(i) # The TCN layers are here. o = TimeDistributed(Dense(13, activation='linear'))(o) model = Model(inputs=[i], outputs=[o]) model.summary() model.compile(loss='mse', optimizer= keras.optimizers.Adam(lr=0.01)) return model
def gen_stcn(seq_num, n_features, dropout, training_t): tcn_inputs = Input(batch_shape=(None, seq_num, n_features)) tcn = TCN(nb_filters=64, kernel_size=2, activation='relu', dropout_rate=0.1, return_sequences=True)(tcn_inputs, training=training_t) # o = 6,64 tcn = TCN(nb_filters=64, kernel_size=2, activation='relu', dropout_rate=0.1, return_sequences=False)(tcn, training=training_t) # o = 6,64 op = Dense(seq_num * n_features)(tcn) model = Model(inputs=tcn_inputs, outputs=op) return model
def create_model(optimizer="Adam", init="he_normal"): # Input layer: i = Input(shape=input_shape) # Temporal convolutional layer o = TCN(nb_filters=nb_filters, kernel_size=filter_size, dilations=dilation_list, padding=padding, use_skip_connections=use_skip_connections, dropout_rate=dropout_rate, activation=activation, kernel_initializer=kernel_initializer, use_batch_norm=use_batch_norm, use_layer_norm=use_layer_norm, return_sequences=False)(i) # Output Layer o = Dense(1, activation=softmax)(o) model = Model(inputs=[i], outputs=[o]) # Compile model model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy']) # history = model.fit(x_train, y_train, # validation_data=[x_val, y_val], # batch_size=params['batch_size'], # epochs=params['epochs'], # verbose=0) # finally we have to make sure that history object and model are returned return model #, history
def defineNetwork(self): numFeatsFlow = int(self.cnnModel.output.shape[1]) # [ b, f, t ] flowInp = Input(shape=(self.flowSteps, numFeatsFlow)) imuModel = self.imuBlock(self.imuShape) merge = concatenate([flowInp, imuModel.outputs[0]]) # [ b, t, f ] y = Permute((2, 1))(merge) y = TCN(nb_filters=128, nb_stacks=3, kernel_size=3, use_skip_connections=True, return_sequences=False, dropout_rate=0.3, dilations=[1, 2, 4, 8])(merge) y = Dense(self.classes, kernel_regularizer=regularizers.l2(0.01), activation='softmax')(y) model = Model([flowInp, imuModel.inputs[0]], y) optimizer = SGD(lr=1e-2, momentum=0.9, decay=1e-4, clipnorm=1., clipvalue=0.5) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc']) return model
def decode(raw, event_id, tmin, tmax): epochs = helper.getEpochs(raw, event_id, tmin=1, tmax=2.5) X = epochs.get_data() #swapping features and time points X = np.transpose( X, (0, 2, 1) ) #prepare X shape to be compatible with LSTM (not sure this is the best way) y = epochs.events[:, -1] cv = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=42) for train, test in cv.split(X, y): X_train, X_test = X[train], X[test] y_train, y_test = y[train], y[test] batch_size, timesteps, input_dim = None, 376, 8 i = Input(batch_shape=(batch_size, timesteps, input_dim)) o = TCN(return_sequences=False)(i) o = Dense(1)(o) m = Model(inputs=[i], outputs=[o]) m.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) history = m.fit(X_train, y_train, validation_split=0.2, epochs=5) score = m.evaluate(X_test, y_test, batch_size=16) print("Accuracy: %.2f%%" % (score[1] * 100))
def get_tcn(seq_len, feature_dim, output_dim=2, nb_filters=8, nb_stacks=3, use_skip_connections=True, return_sequences=True, use_batch_norm=True, dilation_stages=5): dilations = [2**x for x in range(dilation_stages)] i = Input(shape=(seq_len, feature_dim)) x = Reshape(target_shape=(1, seq_len, feature_dim))(i) x = MyConv1D(filters=nb_filters, kernel_size=3, name='initial_conv', kernel_initializer='glorot_normal')(x) x = TCN(nb_filters=nb_filters, kernel_size=3, dilations=dilations, nb_stacks=nb_stacks, use_skip_connections=use_skip_connections, return_sequences=return_sequences, use_batch_norm=use_batch_norm, kernel_initializer='glorot_normal')(x) if return_sequences: x = Reshape(target_shape=(seq_len, nb_filters))(x) else: x = Reshape(target_shape=(nb_filters, ))(x) x = Dense(output_dim)(x) x = Softmax()(x) model = Model(inputs=[i], outputs=[x]) return model
def train(): # Good exercise: https://www.crcv.ucf.edu/data/UCF101.php # replace data() by this dataset. # Useful links: # - https://www.pyimagesearch.com/2019/07/15/video-classification-with-keras-and-deep-learning/ # - https://github.com/sujiongming/UCF-101_video_classification x_train, y_train = data() inputs = Input(shape=(num_frames, h, w, c)) # push num_frames in batch_dim to process all the frames independently of their orders (CNN features). x = Lambda(lambda y: K.reshape(y, (-1, h, w, c)))(inputs) # apply convolutions to each image of each video. x = Conv2D(16, 5)(x) x = MaxPool2D()(x) # re-creates the videos by reshaping. # 3D input shape (batch, timesteps, input_dim) num_features_cnn = np.prod(K.int_shape(x)[1:]) x = Lambda(lambda y: K.reshape(y, (-1, num_frames, num_features_cnn)))(x) # apply the RNN on the time dimension (num_frames dim). x = TCN(16)(x) x = Dense(1, activation='sigmoid')(x) model = Model(inputs=[inputs], outputs=[x]) model.summary() model.compile('adam', 'binary_crossentropy', metrics=['accuracy']) print('Train...') model.fit(x_train, y_train, validation_split=0.2, epochs=5)
def build_model( num_feat, # type: int nb_filters, # type: int kernel_size, # type: int dilations, # type: List[int] nb_stacks, # type: int max_len, # type: int output_len=1, # type: int padding='causal', # type: str use_skip_connections=False, # type: bool return_sequences=True, regression=False, # type: bool dropout_rate=0.2, # type: float name='tcn', # type: str, kernel_initializer='he_normal', # type: str, activation='relu', lr=0.001, use_batch_norm=False, use_layer_norm=False, use_weight_norm=False): input_layer = Input(shape=(max_len, num_feat)) x = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, kernel_initializer, use_batch_norm, use_layer_norm, use_weight_norm, name=name)(input_layer) print('x.shape=', x.shape) x = MaxPooling1D(3)(x) x = Flatten()(x) x = Dropout(dropout_rate)(x) x = Dense(2 * Num_classes, use_bias=False)(x) x = BatchNormalization()(x) x = Activation('relu')(x) x = Dense(Num_classes, use_bias=False)(x) x = BatchNormalization()(x) x = Activation('softmax')(x) output_layer = x model = Model(input_layer, output_layer) model.compile(loss='categorical_crossentropy', optimizer=optimizers.Adam(lr=lr), metrics=['accuracy']) return model
def __init__(self, tcn_out_channel=64, c3d_path='', tcn_path=''): super(C3D_TCN, self).__init__() self.c3d = C3D(in_channels=3) self.tcn = TCN(245760, [128,128,64,tcn_out_channel]) # 245760 == 128, 983040 == 256, 384000 == 160 self.load_models(c3d_path, tcn_path)
def TCN_model(filters, kernel_size, dilations, dropout, a, b): import numpy as np from tensorflow.keras import Sequential from tensorflow.keras.callbacks import Callback from tensorflow.keras.datasets import imdb from tensorflow.keras.layers import Dense, Dropout, Embedding from tensorflow.keras.preprocessing import sequence from tensorflow.keras import optimizers from tensorflow.keras.layers import MaxPooling1D from tcn import TCN model = Sequential() model.add( TCN(nb_filters=filters, kernel_size=kernel_size, dilations=dilations, input_shape=(a, b), return_sequences=False)) #dilations=[1, 2, 4, 8, 16, 32, 64])) model.add(Dropout(dropout)) #model.add(MaxPooling1D(pool_size=2, strides=None, padding='valid', data_format='channels_last')) #model.add(TCN(nb_filters=filters, # kernel_size=kernel_size, # dilations=dilations, # return_sequences=False)) #model.add(Dropout(dropout)) model.add(Dense(1)) #adm= optimizers.Adam(beta_1=0.9, beta_2=0.999, amsgrad=False, decay= 0.005) model.compile(optimizer='adam', loss='mean_squared_error', metrics=['accuracy']) return model
def build_model(X_train, passbands=('g', 'r'), reframe=False, probabilistic=False, nunits=100, bayesian=False, dropout_rate=0.0): if bayesian: mc_dropout = True else: mc_dropout = None npb = len(passbands) inputs = Input(shape=(X_train.shape[1], X_train.shape[2])) hidden = Masking(mask_value=0.)(inputs) hidden = TCN(nunits, return_sequences=True, kernel_size=2, nb_stacks=1, dilations=[1, 2, 4, 8], padding='causal', use_skip_connections=True, dropout_rate=dropout_rate, activation='sigmoid')(hidden, training=mc_dropout) hidden = Dropout(dropout_rate)(hidden, training=mc_dropout) if reframe is True: hidden = LSTM(nunits)(hidden) hidden = Dense(npb)(hidden) else: if probabilistic: hidden = TimeDistributed(Dense(npb * 2))(hidden) else: hidden = TimeDistributed(Dense(npb * 1))(hidden) if probabilistic: outputs = tfp.layers.DistributionLambda( lambda t: tfd.Normal(loc=t[..., :npb], scale=1e-3 + tf.math.softplus(0.01 * t[..., npb:])))(hidden) else: outputs = hidden model = Model(inputs, outputs) return model
def create_tcn_model(num_features, num_classes, loss_func, optimizer='adam', nb_filters=64, kernel_size=2, nb_stacks=1, dilations=[1, 2, 4], dropout_rate=0.2, use_skip_connections=True, use_batch_norm=False, activation='linear'): inp = Input(shape=(num_features, 1), name='input') c = TCN(nb_filters=nb_filters, kernel_size=kernel_size, nb_stacks=nb_stacks, dilations=dilations, use_skip_connections=use_skip_connections, dropout_rate=dropout_rate, activation=activation, use_batch_norm=use_batch_norm)(inp) c = layers.Dense(num_classes)(c) model = Model(inputs=inp, outputs=c) model.compile(optimizer=optimizer, loss=loss_func, metrics=['accuracy']) return model
def __init__(self, in_channels, out_channels, kernel_size, stride=1, dropout=0, residual=True): super(ST_GCN, self).__init__() assert len(kernel_size) == 2 assert kernel_size[0] % 2 == 1 padding = ((kernel_size[0] - 1) // 2, 0) self.gcn = ConvTemporalGraphical(in_channels, out_channels, kernel_size[1]) self.tcn = TCN(out_channels, kernel_size, stride, padding, dropout) if not residual: self.residual = zero elif (in_channels == out_channels) and (stride == 1): self.residual = x1 else: self.residual = nn.Sequential( nn.Conv2d( in_channels, out_channels, kernel_size=1, stride=(stride, 1)), nn.BatchNorm2d(out_channels), ) self.relu = nn.ReLU(inplace=True)
def tcn1(input_shape, logger, nb_filters=128, filter_size=2, optimizer="Adam", init="he_normal"): # Calculate number of blocks: def calc_dilations(filter_size, field): import math max_dil = field / filter_size max_dil = math.ceil(math.log(max_dil) / math.log(2)) dil_list = [2**i for i in range(0, max_dil + 1)] return (dil_list) # TCN params nb_filters = nb_filters filter_size = filter_size dilation_list = calc_dilations(filter_size=filter_size, field=input_shape[1]) padding = "same" use_skip_connections = True dropout_rate = 0.0 activation = "relu" kernel_initializer = "he_normal" use_batch_norm = True use_layer_norm = True nb_stacks = 1 logger.info(f"Dilation list: {dilation_list}") # Input layer: i = Input(shape=input_shape) # Temporal convolutional layer o = TCN(nb_filters=nb_filters, kernel_size=filter_size, dilations=dilation_list, padding=padding, use_skip_connections=use_skip_connections, dropout_rate=dropout_rate, activation=activation, kernel_initializer=kernel_initializer, use_batch_norm=use_batch_norm, use_layer_norm=use_layer_norm, return_sequences=False, nb_stacks=nb_stacks)(i) # Output Layer o = Dense(1, activation="sigmoid")(o) model = Model(inputs=[i], outputs=[o]) # Compile model model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy']) return model
def modelo_CNN3(X_train, y_train, X_test, y_test, individual, epocas): """ Cria um modelo CNN3 :parametro X_train: dados para treinamento :parametro y_train: rótulo dos dados de treinamento :parametro individual: dicionário com os hiperparâmetros do modelo :return: o modelo """ warnings.filterwarnings('ignore') call = [ EarlyStopping(monitor='loss', mode='min', patience=15, verbose=1), ] if individual['filters'] == 0: filters = 16 elif individual['filters'] == 1: filters = 32 else: filters = 64 if individual['norm'] == 0: norm = False else: norm = True if individual['kernel_size'] == 0: kernel_size = 2 elif individual['kernel_size'] == 1: kernel_size = 3 elif individual['kernel_size'] == 2: kernel_size = 5 else: kernel_size = 11 d = [] for i in range(individual['num_conv']): d.append(2**i) i = Input(batch_shape=(None, X_train.shape[1], 1)) o = TCN(nb_filters=filters, kernel_size=kernel_size, nb_stacks=individual['pilhas'], dilations=d, padding='causal', use_skip_connections=False, dropout_rate=individual['dropout'], return_sequences=False, name='tcn')(i) o = Dense(1)(o) model = Model(inputs=[i], outputs=[o]) model.compile(optimizer='Adam', loss='mse') history = model.fit(X_train, y_train, epochs=epocas, verbose=0, batch_size=filters, validation_data=(X_test, y_test), callbacks=call) return model, history
def create_tcn(list_n_filters=[8], kernel_size=4, dilations=[1, 2], nb_stacks=1, activation='norm_relu', n_layers=1, dropout_rate=0.05, use_skip_connections=True, bidirectional=True): if bidirectional: padding = 'same' else: padding = 'causal' dilations = process_dilations(dilations) input_layer = Input(shape=(None, config.N_MELS)) for i in range(n_layers): if i == 0: x = TCN(list_n_filters[i], kernel_size, nb_stacks, dilations, activation, padding, use_skip_connections, dropout_rate, return_sequences=True)(input_layer) else: x = TCN(list_n_filters[i], kernel_size, nb_stacks, dilations, activation, padding, use_skip_connections, dropout_rate, return_sequences=True, name="tcn" + str(i))(x) x = Dense(config.CLASSES)(x) x = Activation('sigmoid')(x) output_layer = x return Model(input_layer, output_layer)
def model_tcn(self): i = Input(shape=(FEED_LEN, 2)) o = TCN(return_sequences=False, activation='relu', nb_filters=128)(i) o = Dense(WINDOW_LEN)(o) mdl = Model(inputs=[i], outputs=[o]) mdl.compile(optimizer='adam', loss='mse', metrics=['mae', 'mape']) mdl.summary() return mdl
def __init__(self, moments, model = 1, data_path = None, batch_size = None, input_dims = 6, trainset = 100, loadModel = None, reporducability = False): if reporducability: np.random.seed(2020) self.batch_size = batch_size save = f"model_{model}+moments_{moments}+batch_size{batch_size}.h5" self.save = ModelCheckpoint(save, save_best_only=True, monitor='val_loss', mode='min') self.moments = moments; self.stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5) self.x, self.y = self.get_data(data_path) if loadModel == None: '''make the model here''' i = Input(batch_shape=(self.batch_size, self.moments, 4)) if model == 1: x1 = TCN(return_sequences=False, nb_filters=(self.moments)*2, dilations=[2**i for i in range(int(np.log2(moments)))], nb_stacks=2, dropout_rate=.3, kernel_size=2)(i) x2 = Lambda(lambda z: backend.reverse(z, axes=-1))(i) x2 = TCN(return_sequences=False, nb_filters=(self.moments)*2, dilations=[2**i for i in range(int(np.log2(moments)))], nb_stacks=2, dropout_rate=.1, kernel_size=2)(x2) x = add([x1, x2]) o = Dense(1, activation='linear')(x) elif model == 2: x1 = TCN(return_sequences=True, nb_filters=(self.moments) * 2, dilations=[2**i for i in range(int(np.log2(moments)))], nb_stacks=2, dropout_rate=.3, kernel_size=2)(i) x2 = Lambda(lambda z: backend.reverse(z, axes=-1))(i) x2 = TCN(return_sequences=True, nb_filters=(self.moments) * 2, dilations=[2**i for i in range(int(np.log2(moments)))], nb_stacks=2, dropout_rate=.1, kernel_size=2)(x2) x = add([x1, x2]) x1 = LSTM(5, return_sequences=False, dropout=.3)(x) x2 = Lambda(lambda z: backend.reverse(z, axes=-1))(x) x2 = LSTM(5, return_sequences=False, dropout=.3)(x2) x = add([x1, x2]) o = Dense(1, activation='linear')(x) elif model == 3: # print([2**i for i in range(int(np.log2(moments) - 1))]) x = TCN(return_sequences=True, nb_filters=32, dilations=[2**i for i in range(int(np.log2(moments)))], nb_stacks=2, dropout_rate=.3, kernel_size=4)(i) x1 = TCN(return_sequences=True, nb_filters = 16, dilations = [2**i for i in range(int(np.log2(moments)))], nb_stacks = 2, dropout_rate=.3, kernel_size=4)(x) x2 = LSTM(32, return_sequences=True, dropout=.3)(i) x2 = LSTM(16, return_sequences=True, dropout=.3)(x2) x = add([x1, x2]) x = Dense(8, activation='linear')(x) x = TCN(return_sequences=True, nb_filters=4, dilations=[1, 2, 4], nb_stacks=1, dropout_rate=.3, kernel_size=2, activation=wave_net_activation)(x) x = concatenate([GlobalMaxPooling1D()(x), GlobalAveragePooling1D()(x)]) o = Dense(1, activation='linear')(x) self.m = Model(inputs=i, outputs=o) else: self.m = load_model(loadModel, custom_objects = {'TCN': TCN, 'wave_net_activation': wave_net_activation}) self.m.summary(); self.m.compile(optimizer='adam', loss='mse')
def build_model(self, input_shape, nb_classes): input_layer = keras.layers.Input(input_shape) num_feat = 30 num_classes = 2 nb_filters = 32 kernel_size = 5 dilations = [2**i for i in range(4)] padding = 'causal' nb_stacks = 1 #max_len=X_train[0:1].shape[1] use_skip_connections = True use_batch_norm = True dropout_rate = 0.05 kernel_initializer = 'he_normal' #lr=0.00 activation = 'linear' use_layer_norm = True return_sequences = true #name='tcn_1' x = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, kernel_initializer, use_batch_norm, use_layer_norm)(input_layer) """ return_sequences=False #name='tcn_1' x = TCN(nb_filters, kernel_size, nb_stacks, dilations, padding, use_skip_connections, dropout_rate, return_sequences, activation, kernel_initializer, use_batch_norm, use_layer_norm)(input_layer) """ output_layer = keras.layers.Dense(nb_classes, activation='sigmoid')(x) model = keras.models.Model(inputs=input_layer, outputs=output_layer) model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy']) print(model.summary()) reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=50, min_lr=0.0001) file_path = self.output_directory + 'best_model' model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', save_best_only=True) self.callbacks = [reduce_lr, model_checkpoint] return model
def generator(): with tf.compat.v1.variable_scope("generator", reuse=tf.compat.v1.AUTO_REUSE): i = Input(batch_shape=(batch_size, 24, 9)) o = TCN(return_sequences=True)(i) # The TCN layers are here. o = Dense(hidden_dim)(o) return o
def vanilla_TCN(self, save_dir,model_name): i = Input(shape=(self.sw_width, self.features)) # m = TCN()(i)\ m = TCN(nb_filters=128, return_sequences=False)(i) # o = TCN(nb_filters=128,return_sequences=False)(i) # m = TCN(nb_filters=64,return_sequences=True)(o) # m = TCN(nb_filters=32,return_sequences=False)(o) m = Dense(1, activation='sigmoid')(m) model = Model(inputs=[i], outputs=[m]) keras.optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) model.compile(optimizer='adam', loss='mse', metrics=['mean_absolute_error']) print(model.summary()) # filepath = "model_{epoch:02d}-{val_loss:.2f}-{val_trend_accuracy:.2f}.hdf5" filepath = "model_{epoch:02d}-{val_loss:.7f}-{val_mean_absolute_error:.7f}.hdf5" checkpoint = ModelCheckpoint(os.path.join(save_dir, filepath), monitor='val_loss', verbose=1, save_best_only=True) history = model.fit(self.train_X, self.train_y, batch_size=512, epochs=self.epochs_num, validation_split=0.2, verbose=self.verbose_set, callbacks=[checkpoint]) # y_pred = model.predict(self.train_X).squeeze(axis = 1) # 降低维度 # y_true = self.train_y # # print(y_pred) # train_results = np.sign(y_pred * y_true) # cor = 0 # for x in train_results: # if x > 0: # cor += 1 # acc = cor * 1.0 / len(train_results) # print("The train acc is %f" % acc) y_pred = model.predict(self.test_X).squeeze(axis=1) # 降低维度 y_true = self.test_y # print(y_pred) test_results = np.sign(y_pred * y_true) cor = 0 for x in test_results: if x > 0: cor += 1 acc = cor * 1.0 / len(test_results) print("The test acc is %f" % acc) # ------------------save_model-----------------# if not os.path.isdir(save_dir): os.makedirs(save_dir) # 再存一次 model_path = os.path.join(save_dir, model_name) model.save(model_path) print('Saved trained model at %s ' % model_path) # --------------------plt loss-------------------# plt.figure(figsize=(8, 8), dpi=200) plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('model train vs validation loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'validation'], loc='upper right') plt.savefig(save_dir +'/' + 'loss.jpg') plt.show()
def recovery(H): with tf.compat.v1.variable_scope("recovery", reuse=tf.compat.v1.AUTO_REUSE): #i = Input(batch_shape=H) o = TCN(return_sequences=True)(H) o = Dense(9)(o) #m = Model(inputs=[i], outputs=[o]) #m.summary() return o
def _TCN(self, inputs, n_outputs, training): '''构建 TCN 模型''' outputs = TCN(inputs, n_outputs, self.num_channels, self.sequence_length, self.kernel_size, self.dropout, is_training=training) return outputs
def build_model(): input = Input(batch_shape=(None, 100, 10)) output = TCN(return_sequences=False, dropout_rate=args.dropout_rate, dilations=(1, 2, 4, 8, 16, 32, 64))(input) # The TCN layers are here. output = Dense(1)(output) model = Model(inputs=[input], outputs=[output]) return model