def findPMD(filepath, outputpath1, outputpath2):
"""
findPMD(filepath, outputpath1, outputpath2)
The main function in PMDfinder.
* filepath: input BED file path.
* outputpath1: the output bed file path.
* outputpath2: the output grange file path.
"""
# load DSS methylation data
methylation = pd.read_csv(filepath,
sep='\t',
comment='t',
header=0,
low_memory=False)
# store the location and the percent methylation
a = list(map(float, methylation['X']))
b = list(map(float, methylation['N']))
meth_ratio = [i / j for i, j in zip(a, b)]
# geno_pos = list(map(float, methylation['pos']))
### Data Conversion
# convert methylation ratio to PMD/non-PMD level (y=4x(1-x))
def methRatio2PMDLevel(meth_ratio):
n = len(meth_ratio)
PMD_level = [0] * n
for i in range(n):
PMD_level[i] = 4 * meth_ratio[i] * (1 - meth_ratio[i])
return PMD_level
PMD_level = methRatio2PMDLevel(meth_ratio)
### Sequential Data Matrix
# Extract sequenctial feature by sliding window
N = len(PMD_level)
X = np.zeros((N - 1023, 1024))
for i in range(N - 1023):
X[i, :] = PMD_level[i:i + 1024]
X = X.astype(np.float32)
### Autoencoder
# latent is the last variable
latent_dim = 8
m = Autoencoder(latent_dim)
m.compile(optimizer='adam', loss=losses.MeanSquaredError())
# fit the model
m.fit(X, X, epochs=5, shuffle=True)
# get the encoded PMD
encoded_slicing_PMD = m.encoder(X).numpy()
### k-means
kmeans = KMeans(n_clusters=2, random_state=22).fit(encoded_slicing_PMD)
final_result = kmeans.labels_
### Post-processing steps
## Remove PMD that is less than 101 bp length
assign1 = [] # index for the location equal to 1
for i in range(len(final_result)):
if final_result[i] == 1:
assign1.append(i)
break_pts1 = [
0
] # index for the break point, the next equal to 1 is more than 1bp
for i in range(1, len(assign1)):
if assign1[i] - assign1[i - 1] > 1:
break_pts1.append(i)
# small_PMD_intervals: identify region that is close with each other
small_PMD_intervals = []
for i in range(1, len(break_pts1)):
if assign1[break_pts1[i] - 1] - assign1[break_pts1[i - 1]] + 1 < 101:
small_PMD_intervals.append(i)
# change the PMD interval with less than 101 to Non-PMD
for interval in small_PMD_intervals:
final_result[assign1[break_pts1[interval -
1]:break_pts1[interval]]] = 0
## Merge PMD that is less than 101 bp from the next one
# This need to check the non-PMD region length
assign2 = []
for i in range(len(final_result)):
if final_result[i] == 0:
assign2.append(i)
break_pts2 = [0]
for i in range(1, len(assign2)):
if assign2[i] - assign2[i - 1] > 1:
break_pts2.append(i)
# small non_PMD intervals
small_non_PMD_intervals = []
for i in range(1, len(break_pts2)):
if assign2[break_pts2[i] - 1] - assign2[break_pts2[i - 1]] + 1 < 51:
small_non_PMD_intervals.append(i)
# change the PMD interval with less than 51 to Non-PMD
for interval in small_non_PMD_intervals:
final_result[assign2[break_pts2[interval -
1]:break_pts2[interval]]] = 1
# file output
output_methylation = methylation[:len(methylation) - 1023].copy()
output_methylation.loc[:,
'PMD_predict'] = pd.DataFrame(final_result)[0].map({
1:
'Non-PMD',
0:
'PMD'
})
output_methylation.to_csv(outputpath1, sep='\t', index=False, header=True)
# output grange file
df = pd.DataFrame(columns=['chr', 'start', 'end', 'status'])
ncols = len(output_methylation)
i, j = 0, 0
while i < ncols:
if j == ncols:
df = df.append(
{
'chr': output_methylation.iloc[i, 0],
'start': output_methylation.iloc[i, 1],
'end': output_methylation.iloc[j - 1, 1],
'status': ti
},
ignore_index=True)
break
ti = output_methylation.iloc[i, 4]
tj = output_methylation.iloc[j, 4]
if tj == ti:
j += 1
else:
df = df.append(
{
'chr': output_methylation.iloc[i, 0],
'start': output_methylation.iloc[i, 1],
'end': output_methylation.iloc[j - 1, 1],
'status': ti
},
ignore_index=True)
i = j
df.to_csv(outputpath2, sep='\t', index=False, header=True)
# print(df)
print("Finished PMDfinder!")
def pipeline():
featurearr, simarr, labelarr=load_data()
xarr, yarr, aarr, edge_attrarr=graphdatageneration(featurearr, simarr, labelarr)
dataset = MyDataset(xarr,yarr,aarr,edge_attrarr)
np.random.seed(10)
# Train/test split
idxs = np.random.permutation(len(dataset))
split = int(0.8 * len(dataset))
idx_tr, idx_te = np.split(idxs, [split])
dataset_tr, dataset_te = dataset[idx_tr], dataset[idx_te]
loader_tr = DisjointLoader(dataset_tr, batch_size=32, epochs=30,shuffle=True)
loader_te = DisjointLoader(dataset_te, batch_size=32, epochs=1,shuffle=True)
model=buildmodel(dataset)
opt = optimizers.Adam(lr=learning_rate)
loss_fn = losses.MeanSquaredError()
@tf.function(input_signature=loader_tr.tf_signature(), experimental_relax_shapes=True)
def train_step(inputs, target):
with tf.GradientTape() as tape:
predictions = model(inputs, training=True)
loss = loss_fn(target, predictions)
mae=losses.MeanAbsoluteError()(target, predictions)
mape=losses.MeanAbsolutePercentageError()(target, predictions)
loss += sum(model.losses)
gradients = tape.gradient(loss, model.trainable_variables)
opt.apply_gradients(zip(gradients, model.trainable_variables))
return loss,mae,mape
print("training")
current_batch = 0
model_loss = 0
total_mape=0
total_mae=0
for batch in loader_tr:
outs,mae,mape= train_step(*batch)
model_loss += outs
total_mae+=mae
total_mape+=mape
current_batch += 1
if current_batch == loader_tr.steps_per_epoch:
print("MSE: {}".format(model_loss / loader_tr.steps_per_epoch),
"MAE: {}".format(total_mae/ loader_tr.steps_per_epoch),
"MAPE: {}".format(total_mape/ loader_tr.steps_per_epoch))
model_loss = 0
total_mae = 0
total_mape = 0
current_batch = 0
print("testing")
model_loss = 0
model_mae=0
model_mape = 0
for batch in loader_te:
inputs, target = batch
predictions = model(inputs, training=False)
model_loss += loss_fn(target, predictions)
model_mae += losses.MeanAbsoluteError()(target, predictions)
model_mape+= losses.MeanAbsolutePercentageError()(target, predictions)
model_loss /= loader_te.steps_per_epoch
model_mae /= loader_te.steps_per_epoch
model_mape /= loader_te.steps_per_epoch
print("Done. Test MSE: {}".format(model_loss),
"Test MAE: {}".format(model_mae),
"Test MAPE: {}".format(model_mape))
model.save('/home/som/lab/seed-yzj/newpaper4/laboratory/model/fusion.hdf5')
def __init__(self, output_len, model_parameters):
'''
output_len = length of target vector
model_parameters = model hyper parameters pulled from parameters.py
'''
super(EARSHOT, self).__init__(name='earshot')
self.model_parameters = model_parameters
self.mask = Masking(mask_value=-9999, name="mask")
if self.model_parameters.hidden['type'] == "LSTM":
self.hidden = LSTM(self.model_parameters.hidden['size'],
return_sequences=True,
stateful=False,
name="LSTM")
elif self.model_parameters.hidden['type'] == "GRU":
self.hidden = GRU(self.model_parameters.hidden['size'],
return_sequences=True,
name="GRU")
# loss function and output activation are coupled, this sets them both
if self.model_parameters.train_loss == 'CE':
self.loss = losses.BinaryCrossentropy(from_logits=True)
self.activation = tf.nn.sigmoid
elif self.model_parameters.train_loss == 'MSE':
self.loss = losses.MeanSquaredError()
self.activation = tf.nn.tanh
# set learning rate schedule
if list(self.model_parameters.learning_schedule.keys())[0] == 'noam':
self.lr_sched = noam_decay_lr(
self.model_parameters.learning_schedule['noam']['warmup'])
lr = self.model_parameters.learning_schedule['noam']['initial']
elif list(self.model_parameters.learning_schedule.keys()
)[0] == 'constant':
self.lr_sched = constant_lr(
self.model_parameters.learning_schedule['constant']['rate'])
lr = self.model_parameters.learning_schedule['constant']['rate']
elif list(self.model_parameters.learning_schedule.keys()
)[0] == 'polynomial':
self.lr_sched = polynomial_decay_lr(
self.model_parameters.learning_schedule['polynomial']
['max_epochs'],
self.model_parameters.learning_schedule['polynomial']
['poly_pow'])
lr = self.model_parameters.learning_schedule['polynomial'][
'initial']
elif list(self.model_parameters.learning_schedule.keys())[0] == 'step':
self.lr_sched = step_decay_lr(
self.model_parameters.learning_schedule['step']['initial'],
self.model_parameters.learning_schedule['step']['drop_factor'],
self.model_parameters.learning_schedule['step']['drop_every'])
lr = self.model_parameters.learning_schedule['step']['initial']
# optimizer
if list(self.model_parameters.optimizer.keys())[0] == 'ADAM':
self.optimizer = tf.keras.optimizers.Adam(
learning_rate=lr, **self.model_parameters.optimizer['ADAM'])
elif list(self.model_parameters.optimizer.keys())[0] == 'SGD':
self.optimizer = tf.keras.optimizers.SGD(
learning_rate=lr, **self.model_parameters.optimizer['SGD'])
self.dense_output = Dense(output_len, activation=self.activation)
from machine_learning_solver.PINN import AllenCahnPINN
import tensorflow.keras.losses as losses
import numpy as np
from sklearn.metrics import mean_squared_error
from numerical_solvers.AllenCahn_FTCS import AllenCahnFTCS
from util.generate_plots import generate_contour_and_snapshots_plot
# Part 1: solve AC equation using PINN
pinn = AllenCahnPINN(loss_obj=losses.MeanSquaredError(),
n_nodes=20,
n_layers=8)
pinn.generate_training_data(n_initial=50, n_boundary=25, equidistant=False)
pinn.perform_training(max_n_epochs=1000,
min_mse=0.05,
track_losses=True,
batch_size='full')
# Print loss data frame and plot solution
print(pinn.loss_df)
generate_contour_and_snapshots_plot(pinn.u_pred,
train_feat=pinn.train_feat,
legend_loc='center left')
# generate_contour_and_snapshots_plot(pinn.u_pred, train_feat=pinn.train_feat, legend_loc='center left',
# savefig_path='plots/Fig10_PINN_contour_and_snapshots_plot.jpg')
# Part 2: use initial initial condition predicted by PINN for FTCS
# Generate initial data for Upwind solver
n_spatial = 512
n_temporal = 201
def coolLinear(x):
return 3 * x + 1
# generate training data
bounds = (-10, 10) # represents full system dynamics
inputList, outputList = randomPoints(10000, bounds, boolfunc)
# neural network code
model = models.Sequential()
model.add(layers.Dense(1, activation='linear', input_shape=(1, )))
model.compile(optimizer='Adam',
loss=losses.MeanSquaredError(),
metrics=['mean_squared_error'])
history = model.fit(np.array(inputList), np.array(outputList), epochs=200)
print(model.get_weights())
# plots out learning curve
# plt.plot(history.history['mean_squared_error'], label='mean_squared_error')
# plt.xlabel('Epoch')
# plt.ylabel('MSE')
# plt.ylim([0.0, 0.2])
# plt.legend(loc='lower right')
# plt.show()
bounds = (-15, 15) # represents full system dynamics
sub_dir = datetime.strftime(datetime.now(), r'%Y%m%d-%H%M%S')
root_dir = 'tmp/'
s1 = 'use_learnable' if USE_LEARNABLE else 'use_fixed'
s2 = 'use_recon' if USE_RECONSTRUCT else 'no_recon'
sub_dir = s1 + '_' + s2 + '/' + sub_dir
writer = tf.summary.create_file_writer(root_dir + sub_dir)
initer = k.initializers.RandomUniform(0.2, 1.0)
weights: List[tf.Variable] = [
tf.Variable(initer([], tf.float32), trainable=USE_LEARNABLE)
for i in range(3)
]
optimizer = k.optimizers.Adam(0.0001)
ce_fn1 = kls.CategoricalCrossentropy()
ce_fn2 = kls.CategoricalCrossentropy()
mse_fn = kls.MeanSquaredError()
ceacc_fn1 = k.metrics.CategoricalAccuracy()
ceacc_fn2 = k.metrics.CategoricalAccuracy()
batch_size = 256
epochs = 10
@tf.function
def step(x, y1, y2, y3):
with tf.GradientTape() as tape:
p1, p2, p3 = model(x, training=True)
l1 = ce_fn1(y1, p1)
ceacc_fn1.update_state(y1, p1)
l1_w = uncertaint_weight(l1, weights[0])
l2 = ce_fn2(y2, p2)
ceacc_fn2.update_state(y2, p2)
l2_w = uncertaint_weight(l2, weights[1])
hidden_dense_layer3 = layers.Dense(
16, activation=activations.tanh)(hidden_batch_layer2)
hidden_batch_layer3 = layers.BatchNormalization()(hidden_dense_layer3)
output_para = layers.Dense(
env.action_dim, activation=activations.sigmoid)(hidden_batch_layer3)
output_hidden = layers.Dense(4, activation=activations.tanh)(output_para)
output_eval = layers.Dense(1,
activation=activations.sigmoid)(output_hidden)
model = models.Model(inputs=[input_layer],
outputs=[output_para, output_eval])
return model
model = create_model()
optimizer = optimizers.RMSprop(0.00001)
loss_object = losses.MeanSquaredError()
def discount_rewards(rewards, r=0.04):
discounted_rewards = np.zeros_like(rewards)
running_add = 0
for t in reversed(range(0, len(rewards))):
running_add = running_add / (1 + r / 250) + rewards[t]
discounted_rewards[t] = running_add
discounted_rewards -= np.mean(discounted_rewards)
discounted_rewards /= np.std(discounted_rewards)
return discounted_rewards
def interaction_process():
ob = env.reset()
def build_model(local_bm_hyperparameters, local_bm_settings):
model_built = 0
time_steps_days = int(local_bm_hyperparameters['time_steps_days'])
epochs = int(local_bm_hyperparameters['epochs'])
batch_size = int(local_bm_hyperparameters['batch_size'])
workers = int(local_bm_hyperparameters['workers'])
optimizer_function = local_bm_hyperparameters['optimizer']
optimizer_learning_rate = local_bm_hyperparameters['learning_rate']
if optimizer_function == 'adam':
optimizer_function = optimizers.Adam(optimizer_learning_rate)
elif optimizer_function == 'ftrl':
optimizer_function = optimizers.Ftrl(optimizer_learning_rate)
losses_list = []
loss_1 = local_bm_hyperparameters['loss_1']
loss_2 = local_bm_hyperparameters['loss_2']
loss_3 = local_bm_hyperparameters['loss_3']
union_settings_losses = [loss_1, loss_2, loss_3]
if 'mape' in union_settings_losses:
losses_list.append(losses.MeanAbsolutePercentageError())
if 'mse' in union_settings_losses:
losses_list.append(losses.MeanSquaredError())
if 'mae' in union_settings_losses:
losses_list.append(losses.MeanAbsoluteError())
if 'm_mape' in union_settings_losses:
losses_list.append(modified_mape())
if 'customized_loss_function' in union_settings_losses:
losses_list.append(customized_loss())
metrics_list = []
metric1 = local_bm_hyperparameters['metrics1']
metric2 = local_bm_hyperparameters['metrics2']
union_settings_metrics = [metric1, metric2]
if 'rmse' in union_settings_metrics:
metrics_list.append(metrics.RootMeanSquaredError())
if 'mse' in union_settings_metrics:
metrics_list.append(metrics.MeanSquaredError())
if 'mae' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsoluteError())
if 'mape' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsolutePercentageError())
l1 = local_bm_hyperparameters['l1']
l2 = local_bm_hyperparameters['l2']
if local_bm_hyperparameters['regularizers_l1_l2'] == 'True':
activation_regularizer = regularizers.l1_l2(l1=l1, l2=l2)
else:
activation_regularizer = None
nof_features_for_training = local_bm_hyperparameters[
'nof_features_for_training']
# creating model
forecaster_in_block = tf.keras.Sequential()
print('creating the ANN model...')
# first layer (DENSE)
if local_bm_hyperparameters['units_layer_1'] > 0:
forecaster_in_block.add(
layers.Dense(
units=local_bm_hyperparameters['units_layer_1'],
activation=local_bm_hyperparameters['activation_1'],
input_shape=(local_bm_hyperparameters['time_steps_days'],
nof_features_for_training),
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(local_bm_hyperparameters['dropout_layer_1'])))
# second LSTM layer
if local_bm_hyperparameters[
'units_layer_2'] > 0 and local_bm_hyperparameters[
'units_layer_1'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.LSTM(
units=local_bm_hyperparameters['units_layer_2'],
activation=local_bm_hyperparameters['activation_2'],
activity_regularizer=activation_regularizer,
dropout=float(local_bm_hyperparameters['dropout_layer_2']),
return_sequences=False)))
forecaster_in_block.add(
RepeatVector(local_bm_hyperparameters['repeat_vector']))
# third LSTM layer
if local_bm_hyperparameters['units_layer_3'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.LSTM(
units=local_bm_hyperparameters['units_layer_3'],
activation=local_bm_hyperparameters['activation_3'],
activity_regularizer=activation_regularizer,
dropout=float(local_bm_hyperparameters['dropout_layer_3']),
return_sequences=True)))
if local_bm_hyperparameters['units_layer_4'] == 0:
forecaster_in_block.add(
RepeatVector(local_bm_hyperparameters['repeat_vector']))
# fourth layer (DENSE)
if local_bm_hyperparameters['units_layer_4'] > 0:
forecaster_in_block.add(
layers.Dense(units=local_bm_hyperparameters['units_layer_4'],
activation=local_bm_hyperparameters['activation_4'],
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(local_bm_hyperparameters['dropout_layer_4'])))
# final layer
forecaster_in_block.add(
TimeDistributed(layers.Dense(units=nof_features_for_training)))
forecaster_in_block.save(''.join(
[local_bm_settings['models_path'], 'in_block_NN_model_structure_']),
save_format='tf')
forecast_horizon_days = local_bm_settings['forecast_horizon_days']
forecaster_in_block.build(input_shape=(1, forecast_horizon_days + 1,
nof_features_for_training))
forecaster_in_block.compile(optimizer=optimizer_function,
loss=losses_list,
metrics=metrics_list)
forecaster_in_block_json = forecaster_in_block.to_json()
with open(
''.join([
local_bm_settings['models_path'],
'freq_acc_forecaster_in_block.json'
]), 'w') as json_file:
json_file.write(forecaster_in_block_json)
json_file.close()
print(
'build_model function finish (model structure saved in json and ts formats)'
)
return True, model_built
def __init__(self, root_data_dir, k_folds, selected_fold, augment_level,
batch_size, slices, seed, experiment_dir):
if root_data_dir[-1] == '/':
utils.log('Removing extra \'/\' from root_data_dir')
root_data_dir = root_data_dir[:-1]
utils.log('Creating CNN with parameters:\n' +
'\troot_data_dir={},\n'.format(root_data_dir) +
'\tk_folds={},\n'.format(k_folds) +
'\tselected_fold={},\n'.format(selected_fold) +
'\taugment_level={},\n'.format(augment_level) +
'\tbatch_size={},\n'.format(batch_size) +
'\tslices={},\n'.format(slices) +
'\tseed={},\n'.format(seed) +
'\texperiment_dir={}'.format(experiment_dir))
self.experiment_dir = experiment_dir
self.checkpoints_dir = join(self.experiment_dir, 'checkpoints')
self.log_dir = join(
self.experiment_dir,
'logs-{}'.format(datetime.now().strftime('%d-%m-%Y_%Hh%Mm%Ss')))
self.cross_sections_dir = join(self.experiment_dir, 'cross_sections')
self.enfaces_dir = join(self.experiment_dir, 'enfaces')
makedirs(self.cross_sections_dir, exist_ok=True)
makedirs(self.enfaces_dir, exist_ok=True)
self.writer = tf.summary.create_file_writer(self.log_dir)
# build model
self.model = get_unet(IMAGE_DIM, IMAGE_DIM // slices)
self.optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)
self.model.compile(optimizer=self.optimizer,
loss=losses.MeanSquaredError())
# set up checkpoints
self.epoch = tf.Variable(0)
self.checkpoint = tf.train.Checkpoint(model=self.model,
optimizer=self.optimizer,
epoch=self.epoch)
self.manager = tf.train.CheckpointManager(
self.checkpoint,
directory=self.checkpoints_dir,
max_to_keep=None #keep all checkpoints
)
if self.manager.latest_checkpoint:
utils.log('Loading latest checkpoint {}'.format(
self.manager.latest_checkpoint))
self.restore_status = self.checkpoint.restore(
self.manager.latest_checkpoint)
else:
self.restore_status = None
if augment_level == AUGMENT_NORMALIZE:
self.mean = STATS[basename(
root_data_dir)][seed][k_folds][selected_fold]['mean']
self.std = STATS[basename(
root_data_dir)][seed][k_folds][selected_fold]['std']
utils.log('mean={}, std={}'.format(self.mean, self.std))
else:
self.use_random_jitter = (augment_level == AUGMENT_FULL)
utils.log('Using augmented data, not fetching mean/std')
utils.log('use_random_jitter={}'.format(self.use_random_jitter))
self.data_loaded = False
self.root_data_dir = root_data_dir
self.k_folds = k_folds
self.selected_fold = selected_fold
self.augment_level = augment_level
self.batch_size = batch_size
self.slices = slices
self.seed = seed
def compile_fit(self,
model_input,
q_train_padded,
a_train_padded,
y_q_label_df,
y_a_label_df,
y_q_classify_list,
y_q_classify_dict,
y_a_classify_list,
y_a_classify_dict,
epoch_num=3):
"""
This function is used to switch between numrical. The switch controled by hyperparameters self.TYPE
When self.TYPE == 'num', input will be q_train_padded and y_q_label_df (others are same)
Meanwhile, switch to ['MSE'] as loss and ['mse', 'mae'] as metrics
When self.TYPE == 'classify', input will be q_train_padded and y_q_classify_list[0] etc.
Meanwhile, swith to ['categorical_crossentropy'] as loss and ['accuracy'] as metrics
"""
start_time = time()
print("*" * 40, "Start {} Processing".format(model_input._name),
"*" * 40)
# loss_fun = 'categorical_crossentropy'
# loss_fun = 'MSE' #MeanSquaredError
# loss_fun = '
METRICS = [
metrics.TruePositives(name='tp'),
metrics.FalsePositives(name='fp'),
metrics.TrueNegatives(name='tn'),
metrics.FalseNegatives(name='fn'),
metrics.CategoricalAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.AUC(name='auc'),
# F1Score(num_classes = int(y_train.shape[1]), name='F1')
]
loss_fun = None
metrics_fun = None
# becase large data input, we want to process automaticaly. So set this arugs to choose
# question process or answer process automatically
if self.PART == 'q':
print("Start processing question part")
# start to decide complie parameters
if self.TYPE == 'num':
print("Start numerical output")
# call split
X_train, X_val, y_train, y_val = self.split_data(
q_train_padded, y_q_label_df, test_size=0.2)
loss_fun = losses.MeanSquaredError()
metrics_fun = ['mse', 'mae']
elif self.TYPE == 'classify':
print("Start classify output")
X_train, X_val, y_train, y_val = self.split_data(
q_train_padded, y_q_classify_list[0], test_size=0.2)
loss_fun = losses.CategoricalCrossentropy()
metrics_fun = METRICS
else:
print("UNKNOW self.TYPE")
elif self.PART == 'a':
print("Start processing answer part")
if self.TYPE == 'num':
print("Start numerical output")
# call split
X_train, X_val, y_train, y_val = self.split_data(
a_train_padded, y_a_label_df, test_size=0.2)
loss_fun = losses.MeanSquaredError()
metrics_fun = ['mse', 'mae']
elif self.TYPE == 'classify':
print("Start classify output")
X_train, X_val, y_train, y_val = self.split_data(
a_train_padded, y_a_classify_list[0], test_size=0.2)
loss_fun = losses.CategoricalCrossentropy()
metrics_fun = METRICS
else:
print("UNKNOW self.TYPE")
learning_rate = 1e-3
opt_adam = optimizers.Adam(lr=learning_rate, decay=1e-5)
model_input.compile(loss=loss_fun,
optimizer=opt_adam,
metrics=metrics_fun)
# batch_size is subjected to my GPU and GPU memory, after testing, 32 is reasonable value size.
# If vector bigger, this value should dercrease
history = model_input.fit(
X_train,
y_train,
validation_data=(X_val, y_val),
epochs=epoch_num,
batch_size=16,
verbose=1,
callbacks=[PredictCallback(X_val, y_val, model_input)])
# spearmanr_list = PredictCallback(X_val, y_val, model_input).spearmanr_list
# dic = ['loss', 'accuracy', 'val_loss','val_accuracy']
history_dict = [x for x in history.history]
# model_input.predict(train_features[:10])
cost_time = round((time() - start_time), 4)
print("*" * 40,
"End {} with {} seconds".format(model_input._name, cost_time),
"*" * 40,
end='\n\n')
return history, model_input
Y = tf.reshape(Y, [-1, 2048])
#loop through entire file
opt = tf.keras.optimizers.Adam(learning_rate=5e-9)
opt2 = tf.keras.optimizers.Adam(learning_rate=5e-4)
opt3 = tf.keras.optimizers.Adam(learning_rate=5e-4)
#skipping model.round_weights(128)
#the scale factor?
callback2 = tf.keras.callbacks.EarlyStopping(monitor='loss',
min_delta=.01,
patience=30)
model.compile(optimizer=opt, loss=losses.MeanSquaredError())
history = model.fit(X, X, epochs=100000000, callbacks=[callback2])
model.freeze_decoder()
nc = 7
np.set_printoptions(threshold=np.inf)
print(model.layers[1].weights)
print("before clustering")
cluster_weights = tfmot.clustering.keras.cluster_weights
CentroidInitialization = tfmot.clustering.keras.CentroidInitialization
clustering_params = {
np.random.seed(77)
tf_rnd.set_seed(77)
random.seed(77)
## Параметры размеров окна matplotlib
rcParams['figure.figsize'] = (10.0, 5.0)
x_arr = np.arange(-10, 10, 0.5)
y_arr = np.tile([1, 1, 1, 1, 0, 0, 0, 0], 5)
plt.plot(x_arr, y_arr, '-', color='red', label='Real values', markersize=5)
plt.show()
plt.close()
threshold = 0.0001
keras_lf = losses.MeanSquaredError()
class signal_nn:
def __init__(self, neurons=50, activation='relu', threshold=0.000001):
self.threshold = threshold
self.model = models.Sequential()
self.model.add(
layers.Dense(neurons, activation=activation, input_dim=1))
self.model.add(layers.Dense(neurons, activation=activation))
self.model.add(layers.Dense(neurons, activation=activation))
self.model.add(layers.Dense(neurons * 2, activation=activation))
self.model.add(layers.Dense(neurons * 2, activation=activation))
self.model.add(layers.Dense(neurons * 2, activation=activation))
self.model.add(layers.Dense(neurons, activation=activation))
self.model.add(layers.Dense(20, activation=activation))
def create_model(learn_rate, epoch_num, batches, outf_layer, outf_sum,
filter_num, split_filters, which_sum, fc):
input_shape = (98, 98, 3)
inputs = Input(shape=input_shape, name='image_input')
# filter number settings
(f1, f2, f3) = filter_num
# 3 filters for summing
if split_filters:
(f1, f2, f3) = (int(f1 - 3), int(f2 - 3), int(f3))
# normal layer
convolution_1 = Conv2D(f1,
kernel_size=(5, 5),
strides=(1, 1),
activation=outf_layer,
input_shape=input_shape,
name='c_layer_1')(inputs)
s1 = tf.reduce_sum(convolution_1, axis=[1, 2, 3], name='c_layer_1_sum')
pooling_1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name='p_layer_1')(convolution_1)
if split_filters:
# sum "layer"
s1 = tf.reduce_sum(Conv2D(3,
kernel_size=(5, 5),
strides=(1, 1),
activation=outf_sum,
input_shape=input_shape)(inputs),
name='c_layer_1_sum')
convolution_2 = Conv2D(f2,
kernel_size=(5, 5),
strides=(1, 1),
activation=outf_layer,
input_shape=input_shape,
name='c_layer_2')(pooling_1)
s2 = tf.reduce_sum(convolution_2, axis=[1, 2, 3], name='c_layer_2_sum')
pooling_2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
name='p_layer_2')(convolution_2)
if split_filters:
s2 = tf.reduce_sum(Conv2D(3,
kernel_size=(5, 5),
strides=(1, 1),
activation=outf_sum,
input_shape=input_shape)(pooling_1),
name='c_layer_2_sum')
convolution_3 = Conv2D(f3,
kernel_size=(5, 5),
strides=(1, 1),
activation=outf_sum,
input_shape=input_shape,
name='c_layer_3')(pooling_2)
if fc:
flat = Flatten()(convolution_3)
s3 = Dense(1, activation=outf_sum)(flat)
else:
s3 = tf.reduce_sum(convolution_3, axis=[1, 2, 3], name='c_layer_3_sum')
y_pred = s3
for i, s in enumerate([s1, s2]):
if which_sum[i] == 1:
y_pred += s
model = Model(inputs=inputs, outputs=s3)
model.compile(
loss=losses.MeanSquaredError(),
optimizer=optimizers.Adam(learning_rate=learn_rate, name='Adam'),
metrics=[metrics.RootMeanSquaredError(),
metrics.MeanAbsoluteError()])
return model
def train(self, local_settings, local_raw_unit_sales, local_model_hyperparameters, local_time_series_not_improved,
raw_unit_sales_ground_truth):
try:
# data normalization
local_forecast_horizon_days = local_settings['forecast_horizon_days']
local_x_train, local_y_train = build_x_y_train_arrays(local_raw_unit_sales, local_settings,
local_model_hyperparameters,
local_time_series_not_improved)
local_forecast_horizon_days = local_settings['forecast_horizon_days']
local_features_for_each_training = 1
print('starting neural network - individual time_serie training')
# building architecture and compiling model_template
# set training parameters
local_time_steps_days = int(local_settings['time_steps_days'])
local_epochs = int(local_model_hyperparameters['epochs'])
local_batch_size = int(local_model_hyperparameters['batch_size'])
local_workers = int(local_model_hyperparameters['workers'])
local_optimizer_function = local_model_hyperparameters['optimizer']
local_optimizer_learning_rate = local_model_hyperparameters['learning_rate']
if local_optimizer_function == 'adam':
local_optimizer_function = optimizers.Adam(local_optimizer_learning_rate)
elif local_optimizer_function == 'ftrl':
local_optimizer_function = optimizers.Ftrl(local_optimizer_learning_rate)
local_losses_list = []
local_loss_1 = local_model_hyperparameters['loss_1']
local_loss_2 = local_model_hyperparameters['loss_2']
local_loss_3 = local_model_hyperparameters['loss_3']
local_union_settings_losses = [local_loss_1, local_loss_2, local_loss_3]
if 'mape' in local_union_settings_losses:
local_losses_list.append(losses.MeanAbsolutePercentageError())
if 'mse' in local_union_settings_losses:
local_losses_list.append(losses.MeanSquaredError())
if 'mae' in local_union_settings_losses:
local_losses_list.append(losses.MeanAbsoluteError())
if 'm_mape' in local_union_settings_losses:
local_losses_list.append(modified_mape())
if 'customized_loss_function' in local_union_settings_losses:
local_losses_list.append(customized_loss())
if 'pinball_loss_function' in local_union_settings_losses:
local_losses_list.append(pinball_function_loss())
local_metrics_list = []
local_metric1 = local_model_hyperparameters['metrics1']
local_metric2 = local_model_hyperparameters['metrics2']
local_union_settings_metrics = [local_metric1, local_metric2]
if 'rmse' in local_union_settings_metrics:
local_metrics_list.append(metrics.RootMeanSquaredError())
if 'mse' in local_union_settings_metrics:
local_metrics_list.append(metrics.MeanSquaredError())
if 'mae' in local_union_settings_metrics:
local_metrics_list.append(metrics.MeanAbsoluteError())
if 'mape' in local_union_settings_metrics:
local_metrics_list.append(metrics.MeanAbsolutePercentageError())
local_l1 = local_model_hyperparameters['l1']
local_l2 = local_model_hyperparameters['l2']
if local_model_hyperparameters['regularizers_l1_l2'] == 'True':
local_activation_regularizer = regularizers.l1_l2(l1=local_l1, l2=local_l2)
else:
local_activation_regularizer = None
# define callbacks, checkpoints namepaths
local_callback1 = cb.EarlyStopping(monitor='loss',
patience=local_model_hyperparameters['early_stopping_patience'])
local_callbacks = [local_callback1]
print('building current model: Mix_Bid_PeepHole_LSTM_Dense_ANN')
local_base_model = tf.keras.Sequential()
# first layer (DENSE)
if local_model_hyperparameters['units_layer_1'] > 0:
# strictly dim 1 of input_shape is ['time_steps_days'] (dim 0 is number of batches: None)
local_base_model.add(layers.Dense(units=local_model_hyperparameters['units_layer_1'],
activation=local_model_hyperparameters['activation_1'],
input_shape=(local_time_steps_days,
local_features_for_each_training),
activity_regularizer=local_activation_regularizer))
local_base_model.add(layers.Dropout(rate=float(local_model_hyperparameters['dropout_layer_1'])))
# second layer
if local_model_hyperparameters['units_layer_2']:
if local_model_hyperparameters['units_layer_1'] == 0:
local_base_model.add(layers.RNN(
PeepholeLSTMCell(units=local_model_hyperparameters['units_layer_2'],
activation=local_model_hyperparameters['activation_2'],
input_shape=(local_time_steps_days,
local_features_for_each_training),
dropout=float(local_model_hyperparameters['dropout_layer_2']))))
else:
local_base_model.add(layers.RNN(
PeepholeLSTMCell(units=local_model_hyperparameters['units_layer_2'],
activation=local_model_hyperparameters['activation_2'],
dropout=float(local_model_hyperparameters['dropout_layer_2']))))
# local_base_model.add(RepeatVector(local_model_hyperparameters['repeat_vector']))
# third layer
if local_model_hyperparameters['units_layer_3'] > 0:
local_base_model.add(layers.Dense(units=local_model_hyperparameters['units_layer_3'],
activation=local_model_hyperparameters['activation_3'],
activity_regularizer=local_activation_regularizer))
local_base_model.add(layers.Dropout(rate=float(local_model_hyperparameters['dropout_layer_3'])))
# fourth layer
if local_model_hyperparameters['units_layer_4'] > 0:
local_base_model.add(layers.RNN(
PeepholeLSTMCell(units=local_model_hyperparameters['units_layer_4'],
activation=local_model_hyperparameters['activation_4'],
dropout=float(local_model_hyperparameters['dropout_layer_4']))))
local_base_model.add(layers.Dense(units=local_forecast_horizon_days))
# build and compile model
local_base_model.build(input_shape=(1, local_time_steps_days, local_features_for_each_training))
local_base_model.compile(optimizer=local_optimizer_function,
loss=local_losses_list,
metrics=local_metrics_list)
# save model architecture (template for specific models)
local_base_model.save(''.join([local_settings['models_path'],
'generic_forecaster_template_individual_ts.h5']))
local_base_model_json = local_base_model.to_json()
with open(''.join([local_settings['models_path'],
'generic_forecaster_template_individual_ts.json']), 'w') as json_file:
json_file.write(local_base_model_json)
json_file.close()
local_base_model.summary()
# training model
local_moving_window_length = local_settings['moving_window_input_length'] + \
local_settings['moving_window_output_length']
# all input data in the correct type
local_x_train = np.array(local_x_train, dtype=np.dtype('float32'))
local_y_train = np.array(local_y_train, dtype=np.dtype('float32'))
local_raw_unit_sales = np.array(local_raw_unit_sales, dtype=np.dtype('float32'))
# specific time_serie models training loop
local_y_pred_list = []
local_nof_time_series = local_settings['number_of_time_series']
remainder = np.array([time_serie for time_serie in range(local_nof_time_series)
if time_serie not in local_time_series_not_improved])
for time_serie in remainder:
# ----------------------key_point---------------------------------------------------------------------
# take note that each loop the weights and internal last states of previous training are conserved
# that's probably save times and (in aggregated or ordered) connected time series will improve results
# ----------------------key_point---------------------------------------------------------------------
print('training time_serie:', time_serie)
local_x, local_y = local_x_train[:, time_serie: time_serie + 1, :], \
local_y_train[:, time_serie: time_serie + 1, :]
local_x = local_x.reshape(local_x.shape[0], local_x.shape[2], 1)
local_y = local_y.reshape(local_y.shape[0], local_y.shape[2], 1)
# training, saving model and storing forecasts
local_base_model.fit(local_x, local_y, batch_size=local_batch_size, epochs=local_epochs,
workers=local_workers, callbacks=local_callbacks, shuffle=False)
local_base_model.save_weights(''.join([local_settings['models_path'],
'/weights_last_year/_individual_ts_',
str(time_serie), '_model_weights_.h5']))
local_x_input = local_raw_unit_sales[time_serie: time_serie + 1, -local_forecast_horizon_days:]
local_x_input = cof_zeros(local_x_input, local_settings)
local_x_input = local_x_input.reshape(1, local_x_input.shape[1], 1)
print('x_input shape:', local_x_input.shape)
local_y_pred = local_base_model.predict(local_x_input)
print('x_input:\n', local_x_input)
print('y_pred shape:', local_y_pred.shape)
local_y_pred = local_y_pred.reshape(local_y_pred.shape[1])
local_y_pred = cof_zeros(local_y_pred, local_settings)
if local_settings['mini_ts_evaluator'] == "True" and \
local_settings['competition_stage'] != 'submitting_after_June_1th_using_1941days':
mini_evaluator = mini_evaluator_submodule()
evaluation = mini_evaluator.evaluate_ts_forecast(
raw_unit_sales_ground_truth[time_serie, -local_forecast_horizon_days:], local_y_pred)
print('ts:', time_serie, 'with cof_zeros ts mse:', evaluation)
else:
print('ts:', time_serie)
print(local_y_pred)
local_y_pred_list.append(local_y_pred)
local_point_forecast_array = np.array(local_y_pred_list)
local_point_forecast_normalized = local_point_forecast_array.reshape(
(local_point_forecast_array.shape[0], local_point_forecast_array.shape[1]))
local_point_forecast = local_point_forecast_normalized
# save points forecast
np.savetxt(''.join([local_settings['others_outputs_path'], 'point_forecast_NN_LSTM_simulation.csv']),
local_point_forecast, fmt='%10.15f', delimiter=',', newline='\n')
print('point forecasts saved to file')
print('submodule for build, train and forecast time_serie individually finished successfully')
return True
except Exception as submodule_error:
print('train model and forecast individual time_series submodule_error: ', submodule_error)
logger.info('error in training and forecast-individual time_serie schema')
logger.error(str(submodule_error), exc_info=True)
return False
plt.subplot(232)
plt.imshow(pred[0, ..., 0], cmap='gray')
plt.subplot(233)
plt.imshow(self.train_lab[0, ..., 0], cmap='gray')
pred = self.model(self.test_img)
plt.subplot(234)
plt.imshow(self.test_img[0, ..., 0], cmap='gray')
plt.subplot(235)
plt.imshow(pred[0, ..., 0], cmap='gray')
plt.subplot(236)
plt.imshow(self.test_lab[0, ..., 0], cmap='gray')
plt.show()
# %%
# Build model
model = SubPixel(upscale_factor=scale)
model.compile(loss=losses.MeanSquaredError(),
optimizer=optimizers.Adam(learning_rate=0.001),
metrics=[Metric(1).psnr, Metric(1).ssim])
# %%
model.fit(train_ds,
epochs=epochs,
validation_data=val_ds,
callbacks=[PlotCallback()],
verbose=2)
# %%
def __init__(self, scope='MSE'):
super(MSE, self).__init__(scope)
self.cost = losses.MeanSquaredError(reduction=losses.Reduction.SUM)
x = layers.concatenate([x1, x2])
score_output = layers.Dense(1, name="score_output")(x)
class_output = layers.Dense(5, name="class_output")(x)
# 模型构建
model = tf.keras.Model(inputs=[image_input, timeseries_input],
outputs=[score_output, class_output])
# 模型配置
model.compile(
optimizer=optimizers.RMSprop(1e-3),
loss={
"score_output": losses.MeanSquaredError(),
"class_output": losses.CategoricalCrossentropy(from_logits=True)
},
metrics={
"score_output":
[metrics.MeanAbsolutePercentageError(),
metrics.MeanAbsoluteError()],
"class_output": [metrics.CategoricalAccuracy()]
})
tf.print(model.summary())
# 数据构建
img_data = random_sample(size=(100, 32, 32, 3))
ts_data = random_sample(size=(100, 20, 10))
score_targets = random_sample(size=(100, 1))
def train_model(self, local_settings, local_raw_unit_sales,
local_model_hyperparameters):
try:
# loading hyperparameters
local_days_in_focus = local_model_hyperparameters[
'days_in_focus_frame']
local_raw_unit_sales_data = local_raw_unit_sales[:,
-local_days_in_focus:]
local_nof_ts = local_raw_unit_sales.shape[0]
local_forecast_horizon_days = local_settings[
'forecast_horizon_days']
local_features_for_each_training = 1
print(
'starting neural network - individual time_serie training unit_sale_approach'
)
# building architecture and compiling model_template
# set training parameters
local_time_steps_days = int(local_settings['time_steps_days'])
local_epochs = int(local_model_hyperparameters['epochs'])
local_batch_size = int(local_model_hyperparameters['batch_size'])
local_workers = int(local_model_hyperparameters['workers'])
local_optimizer_function = local_model_hyperparameters['optimizer']
local_optimizer_learning_rate = local_model_hyperparameters[
'learning_rate']
local_validation_split = local_model_hyperparameters[
'validation_split']
if local_optimizer_function == 'adam':
local_optimizer_function = optimizers.Adam(
local_optimizer_learning_rate)
elif local_optimizer_function == 'ftrl':
local_optimizer_function = optimizers.Ftrl(
local_optimizer_learning_rate)
local_losses_list = []
local_loss_1 = local_model_hyperparameters['loss_1']
local_loss_2 = local_model_hyperparameters['loss_2']
local_loss_3 = local_model_hyperparameters['loss_3']
local_union_settings_losses = [
local_loss_1, local_loss_2, local_loss_3
]
if 'mape' in local_union_settings_losses:
local_losses_list.append(losses.MeanAbsolutePercentageError())
if 'mse' in local_union_settings_losses:
local_losses_list.append(losses.MeanSquaredError())
if 'mae' in local_union_settings_losses:
local_losses_list.append(losses.MeanAbsoluteError())
if 'm_mape' in local_union_settings_losses:
local_losses_list.append(modified_mape())
if 'customized_loss_function' in local_union_settings_losses:
local_losses_list.append(customized_loss())
if 'pinball_loss_function' in local_union_settings_losses:
local_losses_list.append(pinball_function_loss())
local_metrics_list = []
local_metric1 = local_model_hyperparameters['metrics1']
local_metric2 = local_model_hyperparameters['metrics2']
local_union_settings_metrics = [local_metric1, local_metric2]
if 'rmse' in local_union_settings_metrics:
local_metrics_list.append(metrics.RootMeanSquaredError())
if 'mse' in local_union_settings_metrics:
local_metrics_list.append(metrics.MeanSquaredError())
if 'mae' in local_union_settings_metrics:
local_metrics_list.append(metrics.MeanAbsoluteError())
if 'mape' in local_union_settings_metrics:
local_metrics_list.append(
metrics.MeanAbsolutePercentageError())
local_l1 = local_model_hyperparameters['l1']
local_l2 = local_model_hyperparameters['l2']
if local_model_hyperparameters['regularizers_l1_l2'] == 'True':
local_activation_regularizer = regularizers.l1_l2(l1=local_l1,
l2=local_l2)
else:
local_activation_regularizer = None
# define callbacks, checkpoints namepaths
local_callback1 = cb.EarlyStopping(
monitor='loss',
patience=local_model_hyperparameters['early_stopping_patience']
)
local_callbacks = [local_callback1]
print(
'building current model: individual_time_serie_acc_freq_LSTM_Dense_ANN'
)
local_base_model = tf.keras.Sequential()
# first layer (LSTM)
if local_model_hyperparameters['units_layer_1'] > 0:
local_base_model.add(
layers.LSTM(
units=local_model_hyperparameters['units_layer_1'],
activation=local_model_hyperparameters['activation_1'],
input_shape=(
local_model_hyperparameters['time_steps_days'],
local_features_for_each_training),
dropout=float(
local_model_hyperparameters['dropout_layer_1']),
activity_regularizer=local_activation_regularizer,
return_sequences=True))
# second LSTM layer
if local_model_hyperparameters['units_layer_2'] > 0:
local_base_model.add(
layers.Bidirectional(
layers.LSTM(
units=local_model_hyperparameters['units_layer_2'],
activation=local_model_hyperparameters[
'activation_2'],
activity_regularizer=local_activation_regularizer,
dropout=float(
local_model_hyperparameters['dropout_layer_2']
),
return_sequences=False)))
local_base_model.add(
RepeatVector(local_model_hyperparameters['repeat_vector']))
# third LSTM layer
if local_model_hyperparameters['units_layer_3'] > 0:
local_base_model.add(
layers.Bidirectional(
layers.
RNN(PeepholeLSTMCell(
units=local_model_hyperparameters['units_layer_3'],
dropout=float(
local_model_hyperparameters['dropout_layer_3'])
),
activity_regularizer=local_activation_regularizer,
return_sequences=False)))
local_base_model.add(
RepeatVector(local_model_hyperparameters['repeat_vector']))
# fourth layer (DENSE)
if local_model_hyperparameters['units_layer_4'] > 0:
local_base_model.add(
layers.Dense(
units=local_model_hyperparameters['units_layer_4'],
activation=local_model_hyperparameters['activation_4'],
activity_regularizer=local_activation_regularizer))
local_base_model.add(
layers.Dropout(rate=float(
local_model_hyperparameters['dropout_layer_4'])))
# final layer
local_base_model.add(
layers.Dense(
units=local_model_hyperparameters['units_final_layer']))
# build and compile model
local_base_model.build(
input_shape=(1, local_time_steps_days,
local_features_for_each_training))
local_base_model.compile(optimizer=local_optimizer_function,
loss=local_losses_list,
metrics=local_metrics_list)
# save model architecture (template for specific models)
local_base_model.save(''.join([
local_settings['models_path'],
'_unit_sales_forecaster_template_individual_ts.h5'
]))
local_base_model_json = local_base_model.to_json()
with open(''.join([local_settings['models_path'],
'_unit_sales_forecaster_forecaster_template_individual_ts.json']), 'w') \
as json_file:
json_file.write(local_base_model_json)
json_file.close()
local_base_model.summary()
# training model
local_moving_window_length = local_settings['moving_window_input_length'] + \
local_settings['moving_window_output_length']
# loading x_train and y_train, previously done for third and fourth models trainings
local_builder = local_bxy_x_y_builder()
local_x_train, local_y_train = local_builder.build_x_y_train_arrays(
local_raw_unit_sales, local_settings,
local_model_hyperparameters)
local_x_train = local_x_train.reshape(local_x_train.shape[0],
local_x_train.shape[2],
local_x_train.shape[1])
local_y_train = local_x_train.reshape(local_y_train.shape[0],
local_y_train.shape[2],
local_y_train.shape[1])
# star training time_serie by time_serie
local_y_pred_array = np.zeros(shape=(local_raw_unit_sales.shape[0],
local_forecast_horizon_days),
dtype=np.dtype('float32'))
for time_serie in range(local_nof_ts):
print('training time_serie:', time_serie)
local_x, local_y = local_x_train[:, :, time_serie: time_serie + 1], \
local_y_train[:, :, time_serie: time_serie + 1]
# training, saving model and storing forecasts
local_base_model.fit(local_x,
local_y,
batch_size=local_batch_size,
epochs=local_epochs,
workers=local_workers,
callbacks=local_callbacks,
shuffle=False,
validation_split=local_validation_split)
local_base_model.save_weights(''.join([
local_settings['models_path'],
'/_weights_unit_sales_NN_35_days/_individual_ts_',
str(time_serie), '_model_weights_.h5'
]))
local_x_input = local_raw_unit_sales[
time_serie:time_serie + 1, -local_forecast_horizon_days:]
local_x_input = local_x_input.reshape(1,
local_x_input.shape[1],
1)
# print('x_input shape:', local_x_input.shape)
local_y_pred = local_base_model.predict(local_x_input)
# print('x_input:\n', local_x_input)
# print('y_pred shape:', local_y_pred.shape)
local_y_pred = local_y_pred.reshape(local_y_pred.shape[1])
# print('ts:', time_serie)
# print(local_y_pred)
local_y_pred_array[time_serie:time_serie + 1, :] = local_y_pred
local_point_forecast_normalized = local_y_pred_array.reshape(
(local_y_pred_array.shape[0], local_y_pred_array.shape[1]))
local_point_forecast = local_point_forecast_normalized.clip(0)
# save points forecast
np.save(
''.join([
local_settings['train_data_path'],
'point_forecast_NN_from_unit_sales_training'
]), local_point_forecast)
np.save(
''.join([
local_settings['train_data_path'],
'eleventh_model_NN_unit_sales_forecast_data'
]), local_point_forecast)
np.savetxt(''.join([
local_settings['others_outputs_path'],
'point_forecast_NN_from_unit_sales_training.csv'
]),
local_point_forecast,
fmt='%10.15f',
delimiter=',',
newline='\n')
print('point forecasts saved to file')
print(
'submodule for build, train and forecast time_serie unit_sales individually finished successfully'
)
return True, local_point_forecast
except Exception as submodule_error:
print(
'train model and forecast individual time_series units_sales_ submodule_error: ',
submodule_error)
logger.info(
'error in training and forecast-individual time_serie unit_sales_ schema'
)
logger.error(str(submodule_error), exc_info=True)
return False, []
N = 10000
T = 200
maxlen = T
x, t = toy_problem(N, T)
x_train, x_val, t_train, t_val = \
train_test_split(x, t, test_size=0.2, shuffle=False)
'''
2. モデルの構築
'''
model = RNN(50)
'''
3. モデルの学習
'''
criterion = losses.MeanSquaredError()
optimizer = optimizers.Adam(learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
amsgrad=True)
train_loss = metrics.Mean()
val_loss = metrics.Mean()
def compute_loss(t, y):
return criterion(t, y)
def train_step(x, t):
with tf.GradientTape() as tape:
preds = model(x)
loss = compute_loss(t, preds)
grads = tape.gradient(loss, model.trainable_variables)
def forecast(self, local_mse, local_normalized_scaled_unit_sales,
local_mean_unit_complete_time_serie, local_raw_unit_sales,
local_settings):
try:
print(
'starting high loss (mse in previous LSTM) time_series in-block forecast submodule'
)
# set training parameters
with open(''.join([local_settings['hyperparameters_path'],
'in_block_time_serie_based_model_hyperparameters.json'])) \
as local_r_json_file:
model_hyperparameters = json.loads(local_r_json_file.read())
local_r_json_file.close()
local_time_series_group = np.load(''.join(
[local_settings['train_data_path'], 'time_serie_group.npy']),
allow_pickle=True)
time_steps_days = int(local_settings['time_steps_days'])
epochs = int(model_hyperparameters['epochs'])
batch_size = int(model_hyperparameters['batch_size'])
workers = int(model_hyperparameters['workers'])
optimizer_function = model_hyperparameters['optimizer']
optimizer_learning_rate = model_hyperparameters['learning_rate']
if optimizer_function == 'adam':
optimizer_function = optimizers.Adam(optimizer_learning_rate)
elif optimizer_function == 'ftrl':
optimizer_function = optimizers.Ftrl(optimizer_learning_rate)
losses_list = []
loss_1 = model_hyperparameters['loss_1']
loss_2 = model_hyperparameters['loss_2']
loss_3 = model_hyperparameters['loss_3']
union_settings_losses = [loss_1, loss_2, loss_3]
if 'mape' in union_settings_losses:
losses_list.append(losses.MeanAbsolutePercentageError())
if 'mse' in union_settings_losses:
losses_list.append(losses.MeanSquaredError())
if 'mae' in union_settings_losses:
losses_list.append(losses.MeanAbsoluteError())
if 'm_mape' in union_settings_losses:
losses_list.append(modified_mape())
if 'customized_loss_function' in union_settings_losses:
losses_list.append(customized_loss())
metrics_list = []
metric1 = model_hyperparameters['metrics1']
metric2 = model_hyperparameters['metrics2']
union_settings_metrics = [metric1, metric2]
if 'rmse' in union_settings_metrics:
metrics_list.append(metrics.RootMeanSquaredError())
if 'mse' in union_settings_metrics:
metrics_list.append(metrics.MeanSquaredError())
if 'mae' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsoluteError())
if 'mape' in union_settings_metrics:
metrics_list.append(metrics.MeanAbsolutePercentageError())
l1 = model_hyperparameters['l1']
l2 = model_hyperparameters['l2']
if model_hyperparameters['regularizers_l1_l2'] == 'True':
activation_regularizer = regularizers.l1_l2(l1=l1, l2=l2)
else:
activation_regularizer = None
# searching for time_series with high loss forecast
time_series_treated = []
poor_results_mse_threshold = local_settings[
'poor_results_mse_threshold']
poor_result_time_serie_list = []
nof_features_for_training = 0
for result in local_mse:
if result[1] > poor_results_mse_threshold:
nof_features_for_training += 1
poor_result_time_serie_list.append(int(result[0]))
# nof_features_for_training = local_normalized_scaled_unit_sales.shape[0]
nof_features_for_training = len(poor_result_time_serie_list)
# creating model
forecaster_in_block = tf.keras.Sequential()
print(
'current model for specific high loss time_series: Mix_Bid_PeepHole_LSTM_Dense_ANN'
)
# first layer (DENSE)
if model_hyperparameters['units_layer_1'] > 0:
forecaster_in_block.add(
layers.Dense(
units=model_hyperparameters['units_layer_1'],
activation=model_hyperparameters['activation_1'],
input_shape=(model_hyperparameters['time_steps_days'],
nof_features_for_training),
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(model_hyperparameters['dropout_layer_1'])))
# second LSTM layer
if model_hyperparameters['units_layer_2'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.RNN(PeepholeLSTMCell(
units=model_hyperparameters['units_layer_2'],
activation=model_hyperparameters['activation_2'],
activity_regularizer=activation_regularizer,
dropout=float(
model_hyperparameters['dropout_layer_2'])),
return_sequences=False)))
forecaster_in_block.add(
RepeatVector(model_hyperparameters['repeat_vector']))
# third LSTM layer
if model_hyperparameters['units_layer_3'] > 0:
forecaster_in_block.add(
layers.Bidirectional(
layers.RNN(PeepholeLSTMCell(
units=model_hyperparameters['units_layer_3'],
activation=model_hyperparameters['activation_3'],
activity_regularizer=activation_regularizer,
dropout=float(
model_hyperparameters['dropout_layer_3'])),
return_sequences=False)))
forecaster_in_block.add(
RepeatVector(model_hyperparameters['repeat_vector']))
# fourth layer (DENSE)
if model_hyperparameters['units_layer_4'] > 0:
forecaster_in_block.add(
layers.Dense(
units=model_hyperparameters['units_layer_4'],
activation=model_hyperparameters['activation_4'],
activity_regularizer=activation_regularizer))
forecaster_in_block.add(
layers.Dropout(
rate=float(model_hyperparameters['dropout_layer_4'])))
# final layer
forecaster_in_block.add(
TimeDistributed(layers.Dense(units=nof_features_for_training)))
# forecaster_in_block.saves(''.join([local_settings['models_path'], '_model_structure_']),
# save_format='tf')
forecast_horizon_days = local_settings['forecast_horizon_days']
forecaster_in_block.build(input_shape=(1, forecast_horizon_days,
nof_features_for_training))
forecaster_in_block.compile(optimizer=optimizer_function,
loss=losses_list,
metrics=metrics_list)
forecaster_in_block_json = forecaster_in_block.to_json()
with open(
''.join([
local_settings['models_path'],
'forecaster_in_block.json'
]), 'w') as json_file:
json_file.write(forecaster_in_block_json)
json_file.close()
forecaster_in_block_untrained = forecaster_in_block
print('specific time_serie model initialized and compiled')
nof_selling_days = local_normalized_scaled_unit_sales.shape[1]
last_learning_day_in_year = np.mod(nof_selling_days, 365)
max_selling_time = local_settings['max_selling_time']
days_in_focus_frame = model_hyperparameters['days_in_focus_frame']
window_input_length = local_settings['moving_window_input_length']
window_output_length = local_settings[
'moving_window_output_length']
moving_window_length = window_input_length + window_output_length
nof_years = local_settings['number_of_years_ceil']
# training
# time_serie_data = local_normalized_scaled_unit_sales
nof_poor_result_time_series = len(poor_result_time_serie_list)
time_serie_data = np.zeros(shape=(nof_poor_result_time_series,
max_selling_time))
time_serie_iterator = 0
for time_serie in poor_result_time_serie_list:
time_serie_data[
time_serie_iterator, :] = local_normalized_scaled_unit_sales[
time_serie, :]
time_serie_iterator += 1
if local_settings['repeat_training_in_block'] == "True":
print(
'starting in-block training of model for high_loss time_series in previous model'
)
nof_selling_days = time_serie_data.shape[1]
# nof_moving_windows = np.int32(nof_selling_days / moving_window_length)
remainder_days = np.mod(nof_selling_days, moving_window_length)
window_first_days = [
first_day for first_day in range(0, nof_selling_days,
moving_window_length)
]
length_window_walk = len(window_first_days)
# last_window_start = window_first_days[length_window_walk - 1]
if remainder_days != 0:
window_first_days[
length_window_walk -
1] = nof_selling_days - moving_window_length
day_in_year = []
[
day_in_year.append(last_learning_day_in_year + year * 365)
for year in range(nof_years)
]
stride_window_walk = model_hyperparameters[
'stride_window_walk']
print('defining x_train')
x_train = []
if local_settings['train_model_input_data_approach'] == "all":
[
x_train.append(
time_serie_data[:, day - time_steps_days:day -
window_output_length])
for day in range(time_steps_days, max_selling_time,
stride_window_walk)
]
elif local_settings[
'train_model_input_data_approach'] == "focused":
[
x_train.append(time_serie_data[:, day:day +
time_steps_days])
for last_day in day_in_year[:-1] for day in range(
last_day + window_output_length, last_day +
window_output_length -
days_in_focus_frame, -stride_window_walk)
]
# border condition, take care with last year, working with last data available, yeah really!!
[
x_train.append(
np.concatenate(
(time_serie_data[:, day -
window_output_length:day],
np.zeros(shape=(nof_poor_result_time_series,
time_steps_days -
window_output_length))),
axis=1))
for last_day in day_in_year[-1:] for day in range(
last_day, last_day -
days_in_focus_frame, -stride_window_walk)
]
else:
logging.info(
"\ntrain_model_input_data_approach is not defined")
print('-a problem occurs with the data_approach settings')
return False, None
print('defining y_train')
y_train = []
if local_settings['train_model_input_data_approach'] == "all":
[
y_train.append(time_serie_data[:, day -
time_steps_days:day])
for day in range(time_steps_days, max_selling_time,
stride_window_walk)
]
elif local_settings[
'train_model_input_data_approach'] == "focused":
[
y_train.append(time_serie_data[:, day:day +
time_steps_days])
for last_day in day_in_year[:-1] for day in range(
last_day + window_output_length, last_day +
window_output_length -
days_in_focus_frame, -stride_window_walk)
]
# border condition, take care with last year, working with last data available, yeah really!!
[
y_train.append(
np.concatenate(
(time_serie_data[:, day -
window_output_length:day],
np.zeros(shape=(nof_poor_result_time_series,
time_steps_days -
window_output_length))),
axis=1))
for last_day in day_in_year[-1:] for day in range(
last_day, last_day -
days_in_focus_frame, -stride_window_walk)
]
# if time_enhance is active, assigns more weight to the last time_steps according to enhance_last_stride
if local_settings['time_enhance'] == 'True':
enhance_last_stride = local_settings['enhance_last_stride']
last_elements = []
length_x_y_train = len(x_train)
x_train_enhanced, y_train_enhanced = [], []
enhance_iterator = 1
for position in range(
length_x_y_train - enhance_last_stride,
length_x_y_train, -1):
[
x_train_enhanced.append(x_train[position])
for enhance in range(1, 3 * (enhance_iterator + 1))
]
[
y_train_enhanced.append(y_train[position])
for enhance in range(1, 3 * (enhance_iterator + 1))
]
enhance_iterator += 1
x_train = x_train[:-enhance_last_stride]
[
x_train.append(time_step)
for time_step in x_train_enhanced
]
y_train = y_train[:-enhance_last_stride]
[
y_train.append(time_step)
for time_step in y_train_enhanced
]
# broadcasts lists to np arrays and applies the last pre-training preprocessing (amplification)
x_train = np.array(x_train)
y_train = np.array(y_train)
print('x_train_shape: ', x_train.shape)
if local_settings['amplification'] == 'True':
factor = local_settings[
'amplification_factor'] # factor tuning was done previously
for time_serie_iterator in range(np.shape(x_train)[1]):
max_time_serie = np.amax(
x_train[:, time_serie_iterator, :])
x_train[:, time_serie_iterator, :][x_train[:, time_serie_iterator, :] > 0] = \
max_time_serie * factor
max_time_serie = np.amax(
y_train[:, time_serie_iterator, :])
y_train[:, time_serie_iterator, :][y_train[:, time_serie_iterator, :] > 0] = \
max_time_serie * factor
print('x_train and y_train built done')
# define callbacks, checkpoints namepaths
model_weights = ''.join([
local_settings['checkpoints_path'],
'check_point_model_for_high_loss_time_serie_',
model_hyperparameters['current_model_name'],
"_loss_-{loss:.4f}-.hdf5"
])
callback1 = cb.EarlyStopping(
monitor='loss',
patience=model_hyperparameters['early_stopping_patience'])
callback2 = cb.ModelCheckpoint(model_weights,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks = [callback1, callback2]
x_train = x_train.reshape(
(np.shape(x_train)[0], np.shape(x_train)[2],
np.shape(x_train)[1]))
y_train = y_train.reshape(
(np.shape(y_train)[0], np.shape(y_train)[2],
np.shape(y_train)[1]))
print('input_shape: ', np.shape(x_train))
# train for each time_serie
# check settings for repeat or not the training
forecaster_in_block.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
workers=workers,
callbacks=callbacks,
shuffle=False)
# print summary (informative; but if says "shape = multiple", probably useless)
forecaster_in_block.summary()
forecaster_in_block.save(''.join([
local_settings['models_path'],
'_high_loss_time_serie_model_forecaster_in_block_.h5'
]))
forecaster_in_block.save_weights(''.join([
local_settings['models_path'],
'_weights_high_loss_ts_model_forecaster_in_block_.h5'
]))
print(
'high loss time_series model trained and saved in hdf5 format .h5'
)
else:
forecaster_in_block.load_weights(''.join([
local_settings['models_path'],
'_weights_high_loss_ts_model_forecaster_in_block_.h5'
]))
# forecaster_in_block = models.load_model(''.join([local_settings['models_path'],
# '_high_loss_time_serie_model_forecaster_.h5']))
print('weights of previously trained model loaded')
# compile model and make forecast (not necessary)
# forecaster_in_block.compile(optimizer='adam', loss='mse')
# evaluating model and comparing with aggregated (in-block) LSTM
print('evaluating the model trained..')
time_serie_data = time_serie_data.reshape(
(1, time_serie_data.shape[1], time_serie_data.shape[0]))
x_input = time_serie_data[:, -forecast_horizon_days:, :]
y_pred_normalized = forecaster_in_block.predict(x_input)
# print('output shape: ', y_pred_normalized.shape)
time_serie_data = time_serie_data.reshape(
(time_serie_data.shape[2], time_serie_data.shape[1]))
# print('time_serie data shape: ', np.shape(time_serie_data))
time_serie_iterator = 0
improved_time_series_forecast = []
time_series_not_improved = []
improved_mse = []
for time_serie in poor_result_time_serie_list:
# for time_serie in range(local_normalized_scaled_unit_sales.shape[0]):
y_truth = local_raw_unit_sales[time_serie:time_serie + 1,
-forecast_horizon_days:]
# print('y_truth shape:', y_truth.shape)
# reversing preprocess: rescale, denormalize, reshape
# inverse reshape
y_pred_reshaped = y_pred_normalized.reshape(
(y_pred_normalized.shape[2], y_pred_normalized.shape[1]))
y_pred_reshaped = y_pred_reshaped[
time_serie_iterator:time_serie_iterator + 1, :]
# print('y_pred_reshaped shape:', y_pred_reshaped.shape)
# inverse transform (first moving_windows denormalizing and then general rescaling)
time_serie_normalized_window_mean = np.mean(
time_serie_data[time_serie_iterator,
-moving_window_length:])
# print('mean of this time serie (normalized values): ', time_serie_normalized_window_mean)
local_denormalized_array = window_based_denormalizer(
y_pred_reshaped, time_serie_normalized_window_mean,
forecast_horizon_days)
local_point_forecast = general_mean_rescaler(
local_denormalized_array,
local_mean_unit_complete_time_serie[time_serie],
forecast_horizon_days)
# print('rescaled denormalized forecasts array shape: ', local_point_forecast.shape)
# calculating MSE
# print(y_truth.shape)
# print(local_point_forecast.shape)
local_error_metric_mse = mean_squared_error(
y_truth, local_point_forecast)
# print('time_serie: ', time_serie, '\tMean_Squared_Error: ', local_error_metric_mse)
previous_result = local_mse[:, 1][local_mse[:, 0] ==
time_serie].item()
time_series_treated.append(
[int(time_serie), previous_result, local_error_metric_mse])
if local_error_metric_mse < previous_result:
# print('better results with time_serie specific model training')
print(time_serie, 'MSE improved from ', previous_result,
'to ', local_error_metric_mse)
improved_time_series_forecast.append(int(time_serie))
improved_mse.append(local_error_metric_mse)
else:
# print('no better results with time serie specific model training')
# print('MSE not improved from: ', previous_result, '\t current mse: ', local_error_metric_mse)
time_series_not_improved.append(int(time_serie))
time_serie_iterator += 1
time_series_treated = np.array(time_series_treated)
improved_mse = np.array(improved_mse)
average_mse_in_block_forecast = np.mean(time_series_treated[:, 2])
average_mse_improved_ts = np.mean(improved_mse)
print('poor result time serie list len:',
len(poor_result_time_serie_list))
print('mean_mse for in-block forecast:',
average_mse_in_block_forecast)
print(
'number of time series with better results with this forecast: ',
len(improved_time_series_forecast))
print(
'mean_mse of time series with better results with this forecast: ',
average_mse_improved_ts)
print('not improved time series =', len(time_series_not_improved))
time_series_treated = np.array(time_series_treated)
improved_time_series_forecast = np.array(
improved_time_series_forecast)
time_series_not_improved = np.array(time_series_not_improved)
poor_result_time_serie_array = np.array(
poor_result_time_serie_list)
# store data of (individual-approach) time_series forecast successfully improved and those that not
np.save(
''.join([
local_settings['models_evaluation_path'],
'poor_result_time_serie_array'
]), poor_result_time_serie_array)
np.save(
''.join([
local_settings['models_evaluation_path'],
'time_series_forecast_results'
]), time_series_treated)
np.save(
''.join([
local_settings['models_evaluation_path'],
'improved_time_series_forecast'
]), improved_time_series_forecast)
np.save(
''.join([
local_settings['models_evaluation_path'],
'time_series_not_improved'
]), time_series_not_improved)
np.savetxt(''.join([
local_settings['models_evaluation_path'],
'time_series_forecast_results.csv'
]),
time_series_treated,
fmt='%10.15f',
delimiter=',',
newline='\n')
forecaster_in_block_json = forecaster_in_block.to_json()
with open(''.join([local_settings['models_path'], 'high_loss_time_serie_model_forecaster_in_block.json']), 'w') \
as json_file:
json_file.write(forecaster_in_block_json)
json_file.close()
print('trained model weights and architecture saved')
print('metadata (results, time_serie with high loss) saved')
print(
'forecast improvement done. (high loss time_serie focused) submodule has finished'
)
except Exception as submodule_error:
print('time_series in-block forecast submodule_error: ',
submodule_error)
logger.info(
'error in forecast of in-block time_series (high_loss_identified_ts_forecast submodule)'
)
logger.error(str(submodule_error), exc_info=True)
return False
return True
layers.Dense(latent_dim, activation='relu'),
])
self.decoder = tf.keras.Sequential([
layers.Dense(784, activation='sigmoid'),
layers.Reshape((28, 28))
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
autoencoder = Autoencoder(latent_dim)
autoencoder.compile(optimizer='adam', loss=losses.MeanSquaredError())
autoencoder.fit(x_train,
x_train,
epochs=10,
shuffle=True,
validation_data=(x_test, x_test))
encoded_imgs = autoencoder.encoder(x_test).numpy()
decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()
n = 10
plt.figure(figsize=(20, 4))
for i in range(n):
# display original
ax = plt.subplot(2, n, i + 1)
activation='softmax',
name="class_output")(fc1)
dense2 = layers.Dense(1, activation='sigmoid', name="bounding_box")(
fc1) # later change this into bounding box regression
values = model.predict(image)
values1 = maxpoolmodel.predict(image)
region_array = np.asarray([[[0.0, 0.0, 1.0, 1.0]]], dtype='float32')
roimodel = tf.keras.Model(inputs=(feature_input, roi_input),
outputs=(dense1, dense2))
roimodel.compile(
optimizer=optimizers.RMSprop(1e-3),
loss={
"bounding_box": losses.MeanSquaredError(),
"class_output": losses.CategoricalCrossentropy(),
},
metrics={
"bounding_box": [
metrics.MeanAbsolutePercentageError(),
metrics.MeanAbsoluteError(),
],
"class_output": [metrics.CategoricalAccuracy()],
},
)
roimodel.summary()
values = values.reshape(
1, 1, 5, 5, 1280) # take into account batch size which is first input
region_array = region_array.reshape(1, 1, 1, 4)
output2 = np.array([1])