def generate(
self, V: Variat, input_features_layer: DenseFeatures
) -> Generator[Model, None, None]:
model = tf.keras.Sequential([
input_features_layer,
layers.Dense(158,
activation='selu',
kernel_initializer=initializers.lecun_normal()),
layers.Dropout(0.2),
layers.Dense(168,
activation='swish',
kernel_initializer=initializers.GlorotNormal()),
layers.Dropout(0.2),
layers.Dense(178,
activation='swish',
kernel_initializer=initializers.GlorotNormal()),
layers.Dropout(0.2),
layers.Dense(188,
activation='selu',
kernel_initializer=initializers.lecun_normal()),
layers.Dropout(0.2),
layers.Dense(1,
activation="sigmoid",
kernel_initializer=initializers.lecun_normal())
])
yield model
def begin_insert_layer(self, layer_dim):
# `self.layers[0].get_weights()` -> [weights, bias]
next_units = self.layers[0].get_weights()[0].shape[0]
layer = Dense(
units=next_units,
activation=tf.nn.relu,
kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2),
kernel_initializer=initializers.GlorotNormal(seed=self.seed),
bias_initializer=initializers.Zeros())
layer.build(input_shape=(None, layer_dim))
self.layers.insert(0, layer)
def build_kernel_initializer(args):
if args is None:
return None
kernel_initializer = None
if args.name == 'RandomUniform':
kernel_initializer = initializers.RandomUniform(
args.minval, args.maxval)
if args.name == 'GlorotNormal':
kernel_initializer = initializers.GlorotNormal()
return kernel_initializer
def __init__(self,
input_dim,
output_dim=2,
hidden_dims=None,
l1=0.01,
l2=0.01,
seed=6):
super(PartCoder, self).__init__()
self.l1 = l1
self.l2 = l2
self.seed = seed
# self.layers = NoDependency([])
# self.__dict__['layers'] = []
self.layers = []
_input_dim = input_dim
for i, dim in enumerate(hidden_dims):
layer = Dense(
units=dim,
activation=tf.nn.relu,
kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2),
kernel_initializer=initializers.GlorotNormal(seed=self.seed),
bias_initializer=initializers.Zeros())
layer.build(input_shape=(None, _input_dim))
_input_dim = dim
self.layers.append(layer)
# Final, adding output_layer (latent/reconstruction layer)
layer = Dense(units=output_dim,
activation=tf.nn.sigmoid,
kernel_regularizer=regularizers.l1_l2(l1=self.l1,
l2=self.l2),
kernel_initializer=initializers.GlorotNormal(seed=6),
bias_initializer=initializers.Zeros())
layer.build(input_shape=(None, _input_dim))
self.layers.append(layer)
def _create_bottom_mlp(self):
self._create_bottom_mlp_padding()
self.bottom_mlp_layers = []
for dim in self.bottom_mlp_dims:
kernel_initializer = initializers.GlorotNormal()
bias_initializer = initializers.RandomNormal(stddev=math.sqrt(1. / dim))
if self.data_parallel_bottom_mlp:
kernel_initializer = BroadcastingInitializer(kernel_initializer)
bias_initializer = BroadcastingInitializer(bias_initializer)
l = tf.keras.layers.Dense(dim, activation='relu',
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
self.bottom_mlp_layers.append(l)
def _create_top_mlp(self):
self._create_top_mlp_padding()
self.top_mlp = []
for i, dim in enumerate(self.top_mlp_dims):
if i == len(self.top_mlp_dims) - 1:
# final layer
activation = 'linear'
else:
activation = 'relu'
kernel_initializer = BroadcastingInitializer(initializers.GlorotNormal())
bias_initializer = BroadcastingInitializer(initializers.RandomNormal(stddev=math.sqrt(1. / dim)))
l = tf.keras.layers.Dense(dim, activation=activation,
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer)
self.top_mlp.append(l)
def last_insert_layer(self, layer_dim):
prev_weights, prev_bias = self.layers[len(self.layers) -
1].get_weights()
prev_units = prev_weights.shape[1]
replace_prev_layer = Dense(
units=prev_units,
activation=tf.nn.relu,
kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2),
)
replace_prev_layer.build(input_shape=(None, prev_weights.shape[0]))
replace_prev_layer.set_weights([prev_weights, prev_bias])
added_layer = Dense(
units=layer_dim,
activation=tf.nn.sigmoid,
kernel_regularizer=regularizers.l1_l2(l1=self.l1, l2=self.l2),
kernel_initializer=initializers.GlorotNormal(seed=self.seed),
bias_initializer=initializers.Zeros())
added_layer.build(input_shape=(None, prev_units))
del self.layers[len(self.layers) - 1]
self.layers.append(replace_prev_layer)
self.layers.append(added_layer)
def build_1CNNBase(self, action_space=6, dueling=True):
self.network_size = 256
X_input = Input(shape=(self.REM_STEP*90) )
# X_input = Input(shape=(self.REM_STEP*7,))
input_reshape=(self.REM_STEP,90)
X = X_input
truncatedn_init = initializers.TruncatedNormal(0, 1e-2)
x_init = initializers.GlorotNormal()
y_init = initializers.glorot_uniform()
const_init = initializers.constant(1e-2)
X_reshaped = Reshape(input_reshape)(X_input)
# slice2 = Permute((2,1))(slice2)
#normailsation for each layer
normlayer_0 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.0001,center=False,
scale=True,
beta_initializer="zeros",
gamma_initializer="ones" )
normlayer_1 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.001,center=True,
scale=True,
beta_initializer="zeros",
gamma_initializer="ones")
normlayer_2 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.001,center=True,
scale=True,
beta_initializer="zeros",
gamma_initializer="ones")
normlayer_3 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.001,center=True,
scale=True,
beta_initializer="zeros",
gamma_initializer="ones")
X = normlayer_0(X_reshaped)
# X = TimeDistributed(Dense(128, activation ="relu", kernel_initializer='he_uniform',))(X)
#
# cnn2 = TimeDistributed(Dense(64, kernel_initializer='he_uniform',))(X)
# cnn2 = LeakyReLU(.4)(cnn2)
# cnn2 = MaxPooling1D(2)(cnn2)
# cnn2 = Flatten()(cnn2)
# cnn2 = LocallyConnected1D(filters=64, kernel_initializer='he_uniform', kernel_size=2)(X)
# cnn2 = LeakyReLU(0.3)(cnn2)
# cnn2 = LocallyConnected1D(filters=64, kernel_initializer='he_uniform', kernel_size=2)(X)
# cnn2 = Dense(128,activation="relu", kernel_initializer='he_uniform', )(cnn2)
# cnn2 = Flatten()(cnn2)
# X = Dense(32,
# kernel_initializer='he_uniform',activation ="relu")(X)
# X = normlayer_3(X)
# X = Dense(64,
# kernel_initializer='he_uniform',activation =LeakyReLU(.3))(X)
X = Flatten()(X)
X = Dense(512, activation =LeakyReLU(.1), kernel_initializer=x_init)(X)
X = Dense(256, activation =LeakyReLU(.1), kernel_initializer=x_init)(X)
if dueling:
state_value = Dense(
1, kernel_initializer=x_init , activation ="softmax")(X)
state_value = Lambda(lambda s: K.expand_dims(
s[:, 0], -1), output_shape=(action_space,))(state_value)
action_advantage = Dense(
action_space, kernel_initializer=x_init, activation = "linear") (X)
action_advantage = Lambda(lambda a: a[:, :] - K.mean(
a[:, :], keepdims=True), output_shape=(action_space,))(action_advantage)
X = Add()([state_value, action_advantage])
else:
# Output Layer with # of actions: 2 nodes (left, right)
X = Dense(action_space, activation="relu",
kernel_initializer='he_uniform')(X)
model = Model(inputs=X_input, outputs=X, name='build_TMaxpoolin_3')
model.compile(loss=tf.keras.losses.Huber(delta=2), metrics=['mean_absolute_error','accuracy'] ,optimizer=Adam(
lr=self.learning_rate))
# model.compile(loss="mean_squared_error", optimizer=Adam(lr=0.00025,epsilon=0.01), metrics=["accuracy"])
model.summary()
return model
def build_modelPar1(self, dueling=True):
self.network_size = 256
X_input = Input(shape=(self.REM_STEP*46))
# X_input = Input(shape=(self.REM_STEP*7,))
input_reshape=(self.REM_STEP,1,46)
X = X_input
truncatedn_init = initializers.TruncatedNormal(0, 1e-2)
x_init = initializers.GlorotNormal()
y_init = initializers.glorot_uniform()
const_init = initializers.constant(1e-2)
X_reshaped = Reshape((2,-1))(X_input)
normlayer_0 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.0001,center=True,
scale=True,
beta_initializer="zeros",)
normlayer_1 = LayerNormalization(
axis=-1,trainable=True,epsilon =0.0001,center=True,
scale=True,
beta_initializer="zeros",gamma_initializer="ones")
normlayer_2= LayerNormalization(
axis=-1,trainable=True,epsilon =0.0001,center=True,
scale=True,
beta_initializer="zeros", gamma_initializer="ones")
const_init = initializers.constant(1e-2)
t = Reshape(input_reshape)(X)
cnn2 = (Conv2D(filters=32, activation = "relu", kernel_initializer=x_init, kernel_size=(1), strides =(2,1), padding = "valid"))(t)
cnn2 = normlayer_0(cnn2)
cnn2 = Reshape((1,-1,))(cnn2)
cnn2 = (LocallyConnected1D(filters=64, activation = "relu",kernel_initializer=x_init, kernel_size=(1), strides =(1), padding = "valid"))(cnn2)
# cnn2 = Flatten()(cnn2)
cnn1 = TimeDistributed(Dense(64, activation = "tanh" , kernel_initializer=x_init,))(X_reshaped)
cnn1 = normlayer_1(cnn1)
cnn1 = TimeDistributed(Dense(32, activation="tanh", kernel_initializer=x_init,))(cnn1)
# cnn1 = Flatten()(cnn1)
cnn1 = Reshape((1,-1,))(cnn1)
conc = concatenate([cnn1,cnn2])
f = Flatten()(conc)
w = Dense(512, activation="relu",kernel_initializer=x_init)(f)
w = Dense(256, activation="relu",kernel_initializer=x_init)(w)
state_value = Dense(1, kernel_initializer=x_init)(w)
state_value = Lambda(lambda s: K.expand_dims(
s[:, 0], -1), output_shape=(self.action_space,))(state_value)
action_advantage = Dense(
self.action_space, activation='linear', kernel_initializer=x_init)(w)
action_advantage = Lambda(lambda a: a[:, :] - K.mean(
a[:, :], keepdims=True), output_shape=(self.action_space,))(action_advantage)
out = Add()([state_value, action_advantage])
model = Model([X_input], out, name='TCnn-model_1')
if self.optimizer_model == 'Adam':
optimizer = Adam(lr=self.learning_rate, clipnorm=2.)
elif self.optimizer_model == 'RMSProp':
optimizer = RMSprop(self.learning_rate, 0.99, 0.0, 1e-6,)
else:
print('Invalid optimizer!')
model.compile(loss=tf.keras.losses.Huber(delta=10), metrics=['mean_absolute_error','accuracy'] , optimizer=optimizer)
model.summary()
return model
# Define some variables for ease of change in the model
Stock = 'NVDA' # Stock to predict
checkpoint_path = f"V4models/{Stock} Check point.ckpt"
model_path = f'V4models/{Stock}.tf'
load_path = model_path # CHANGE THIS
checkpoint_dir = os.path.dirname(checkpoint_path)
tolerant_rate = 2.0 # Threshold for holding stock instead of buy/sell
ACTIVATION = 'tanh'
RECURRENT_ACTIVATION = 'sigmoid'
DROPOUT = [0.003, 0.003, 0.003, 0.003]
UNITS = [128, 256, 256, 256]
BATCH_SIZE = 30
EPOCH = 100 # how many times to run at once
SEQ_LEN = 40 # how many days back used to predict, optimal 30/40 for NVDA
initializer = initializers.GlorotNormal() # Xavier Normal initializer
LEARNING_RATE = 0.0007 # not using for now
LEARNING_DECAY = 1e-6 # not using for now
train_data_pct = 0.80 # percentage of data used for training
# opt = tf.keras.optimizers.Adam(lr=LEARNING_RATE, decay=LEARNING_DECAY)
opt = tf.keras.optimizers.Adam()
TODAY = pd.to_datetime('today').to_pydatetime()
offset = max(1, (TODAY.weekday() + 6) % 7 - 3)
timedelta = datetime.timedelta(offset)
START_DATE = '2010-1-1'
LAST_WORKING_DAY = TODAY - timedelta
print('All variables initialized.')
print(
f'Stock to predict: {Stock}. The last working day is {LAST_WORKING_DAY}, getting data from {START_DATE} to {LAST_WORKING_DAY}.'
)
epochs = 100
batch_size = 75
steps_per_epoch = 18
# Model
model = Sequential()
# Input Layer
model.add(Input(shape=(height, width, 3)))
# Conv Layers
model.add(
layers.Conv2D(4, (5, 5),
activation='relu',
kernel_initializer=initializers.GlorotNormal()))
model.add(layers.Conv2D(4, (5, 5), activation='relu'))
model.add(layers.Conv2D(4, (5, 5), activation='relu'))
model.add(layers.Conv2D(4, (5, 5), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(8, (5, 5), activation='relu'))
model.add(layers.Conv2D(8, (5, 5), activation='relu'))
model.add(layers.Conv2D(8, (5, 5), activation='relu'))
model.add(layers.Conv2D(8, (5, 5), activation='relu'))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(32, (5, 5), activation='relu'))
model.add(layers.Conv2D(32, (5, 5), activation='relu'))
model.add(layers.Conv2D(32, (5, 5), activation='relu'))
model.add(layers.Conv2D(32, (5, 5), activation='relu'))
def model(batch_size, k, alpha, lamda):
initializer = initializers.GlorotNormal()
## Encoder
inputs = Layers.Input(shape=[28, 28])
reshape = Layers.Reshape([28, 28, 1])(inputs)
flatten = Layers.Flatten()(inputs)
conv_1 = Layers.Conv2D(6,
kernel_size=5,
padding='valid',
activation='relu',
kernel_initializer=initializer)(reshape)
max_1 = Layers.MaxPool2D(pool_size=2)(conv_1)
conv_2 = Layers.Conv2D(16,
kernel_size=5,
padding='valid',
activation='relu',
kernel_initializer=initializer)(max_1)
max_2 = Layers.MaxPool2D(pool_size=2)(conv_2)
conv_3 = Layers.Conv2D(60,
kernel_size=4,
padding='valid',
activation='relu',
kernel_initializer=initializer)(max_2)
latent = Layers.Flatten()(conv_3)
[non_anchor,
anchor] = Discriminative(alpha=alpha,
batch_size=batch_size,
k=k,
name='discriminative')([flatten, latent])
encoder = Model(inputs=[inputs],
outputs=[non_anchor, anchor, conv_3, latent])
## Decoder
decoder_inputs = Layers.Input(shape=[1, 1, 60])
recon_1 = Layers.Conv2DTranspose(
16,
kernel_size=4,
strides=1,
padding="valid",
activation="relu",
kernel_initializer=initializer)(decoder_inputs)
recon_2 = Layers.Conv2DTranspose(16,
kernel_size=2,
strides=2,
padding="valid",
activation="relu",
kernel_initializer=initializer)(recon_1)
recon_3 = Layers.Conv2DTranspose(6,
kernel_size=5,
strides=1,
padding="valid",
activation="relu",
kernel_initializer=initializer)(recon_2)
recon_4 = Layers.Conv2DTranspose(6,
kernel_size=2,
strides=2,
padding="valid",
activation="relu",
kernel_initializer=initializer)(recon_3)
recon_5 = Layers.Conv2DTranspose(1,
kernel_size=5,
strides=1,
padding="valid",
activation="relu",
kernel_initializer=initializer)(recon_4)
output = Layers.Reshape([28, 28])(recon_5)
decoder = Model(inputs=[decoder_inputs], outputs=[output])
## Autoencoder
_, _, codings, _ = encoder(inputs)
reconstructions = decoder(codings)
autoencoder = Model(inputs=[inputs], outputs=[reconstructions])
## Custom Loss
def custom_loss(lamda):
"""" Wrapper function which calculates the loss function.
Returns a *function* which calculates the complete loss given only the input and target output """
# Latent loss
latent_loss = tf.reduce_sum(non_anchor - anchor)
def overall_loss(y_true, y_pred):
""" Final loss calculation function to be passed to optimizer"""
# Reconstruction loss
reconstruction_loss = lamda * tf.norm(tf.subtract(y_true, y_pred))
# Overall loss
model_loss = reconstruction_loss + latent_loss
return model_loss
return overall_loss
## Compilation, adding loss and optimiser
optimiser = Adam(learning_rate=0.002)
autoencoder.compile(loss=custom_loss(lamda=lamda), optimizer=optimiser)
return autoencoder, encoder, decoder
class Demucs(keras.Model):
"""
Demucs speech enhancement model.
Args:
- chin (int): number of input channels.
- chout (int): number of output channels.
- hidden (int): number of initial hidden channels.
- depth (int): number of layers.
- kernel_size (int): kernel size for each layer.
- stride (int): stride for each layer.
- causal (bool): if false, uses BiLSTM instead of LSTM.
- resample (int): amount of resampling to apply to the input/output.
Can be one of 1, 2 or 4.
- growth (float): number of channels is multiplied by this for every layer.
- max_hidden (int): maximum number of channels. Can be useful to
control the size/speed of the model.
- normalize (bool): if true, normalize the input.
- glu (bool): if true uses GLU instead of ReLU in 1x1 convolutions.
- rescale (float): controls custom weight initialization.
See https://arxiv.org/abs/1911.13254.
- floor (float): stability flooring when normalizing.
"""
@capture_init
def __init__(self,
chin=1,
chout=1,
hidden=48,
depth=5,
kernel_size=8,
stride=4,
causal=True,
resample=4,
growth=2,
max_hidden=10_000,
normalize=True,
glu=True,
rescale=0.1,
floor=1e-3,
name='demucs',
**kwargs):
super().__init__(name=name, **kwargs)
if resample not in [1, 2, 4]:
raise ValueError("Resample should be 1, 2 or 4.")
self.chin = chin
self.chout = chout
self.hidden = hidden
self.depth = depth
self.kernel_size = kernel_size
self.stride = stride
self.causal = causal
self.resample = resample
self.length_calc = LengthCalc(depth, kernel_size, stride)
if resample == 2:
self.upsample = Upsample2()
self.downsample = Downsample2()
elif resample == 4:
self.upsample = keras.Sequential([Upsample2(), Upsample2()], name="upsample4")
self.downsample = keras.Sequential([Downsample2(), Downsample2()], name="downsample4")
else:
self.upsample = None
self.downsample = None
if normalize:
self.normalize = Normalize(floor)
self.encoder = []
self.decoder = []
if rescale:
initializer = initializers.RandomNormal(stddev=rescale)
else:
initializer = initializers.GlorotNormal()
for index in range(depth):
# Encode layer: chin -> hidden
encode = keras.Sequential(name=f"Encode_{index+1}")
encode.add(layers.Conv1D(hidden, kernel_size, strides=stride, activation='relu', input_shape=(None,chin), kernel_initializer=initializer))
if glu:
encode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer))
encode.add(GLU())
else:
encode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer))
self.encoder.append(encode)
# Decode layer: hidden -> chout
decode = keras.Sequential(name=f"Decode_{index+1}")
if glu:
decode.add(layers.Conv1D(hidden*2, 1, kernel_initializer=initializer))
decode.add(GLU())
else:
decode.add(layers.Conv1D(hidden, 1, activation='relu', kernel_initializer=initializer))
decode.add(layers.Conv1DTranspose(chout, kernel_size, strides=stride, kernel_initializer=initializer))
self.decoder.insert(0, decode)
# Update chin, chout, hidden for next depth, growing hidden by growth
chout = hidden
chin = hidden
hidden = min(int(growth * hidden), max_hidden)
self.lstm = BLSTM(bi=not causal)