def train():
slices = s3.list_slices()
ids = list(labels.keys())
np.random.shuffle(ids)
split = int(math.ceil(len(ids) * 0.7))
train = segment(slices, ids[:split])
test = segment(slices, ids[split:])
training_generator = generate(train)
validation_generator = generate(test)
s3_upload = UploadCheckpoint()
early_stop = EarlyStopping(patience=EARLY_STOPPING_PATIENCE)
filepath = 'models/' + MODEL_NAME
checkpoint = ModelCheckpoint(filepath,
monitor='val_acc',
save_best_only=True,
mode='max')
callbacks_list = [checkpoint, s3_upload, early_stop]
model = cnn()
model.fit_generator(generator=training_generator,
steps_per_epoch=len(train) // BATCH_SIZE,
validation_data=validation_generator,
validation_steps=len(test) // BATCH_SIZE,
epochs=EPOCHS,
callbacks=callbacks_list)
def _setup_cnn(self, X, X_seq_len, is_training):
"""
Sets up the convolutional layers.
Args:
X: input sequence
X_seq_len: inputs' sequence lengths
is_training: whether the model is training, a boolean
Returns:
outputs_squeezed: the layer's outputs shaped [batch_size, height, width]
new_seq_len: shaped [batch_size] which are
- either the new sequence lengths (after taking into account subsampling)
if config.use_dynamic_lengths is True
- or the old sequence lengths
"""
X_expanded = tf.expand_dims(X, axis=3)
with tf.variable_scope("cnn"):
cnn = CNN(self.config)
output, new_seq_len = cnn(X_expanded, X_seq_len, is_training)
output_squeezed = tf.squeeze(output, axis=2)
return output_squeezed, new_seq_len
def _setup_cnn(self, X, X_seq_len, is_training):
X_expanded = tf.expand_dims(X, axis=3)
with tf.variable_scope("cnn"):
cnn = CNN(self.config)
output, new_seq_len = cnn(X_expanded, X_seq_len, is_training)
output_squeezed = tf.squeeze(output, axis=2)
return output_squeezed, new_seq_len
def initialize_model(TIME_PERIODS, input_shape, num_sensors, num_classes,
x_train, y_train_hot, x_test, y_train, y_test):
model = models.cnn(time_periods=TIME_PERIODS,
input_shape=input_shape,
number_of_sensors=num_sensors,
number_of_classes=num_classes,
optimizer='adam',
filters=100,
kernel_size=3)
callbacks_list = [
tf.keras.callbacks.ModelCheckpoint(
filepath='../best_models/best_model.{epoch:02d}-{val_loss:.2f}.h5',
monitor='val_loss',
save_best_only=True),
tf.keras.callbacks.EarlyStopping(monitor='acc', patience=1)
]
# TODO Train the model
# Enable validation to use ModelCheckpoint and EarlyStopping callbacks.
history = model.fit(x_train,
y_train_hot,
batch_size=80,
epochs=50,
callbacks=callbacks_list,
validation_split=0.2,
verbose=1)
plots.show_model_history(history)
# plot metrics
plt.plot(history.history['acc'])
plt.show()
plt.plot(history.history['mean_squared_error'])
plt.show()
y_pred_train, max_y_pred_train = train_model(model, x_train, y_train)
y_pred_test, max_y_pred_test = test_model(model, x_test, y_test)
return y_pred_train, max_y_pred_train, y_pred_test, max_y_pred_test
[input_data_scaled_no_dummies, input_data_dummy], axis=1)
input_data, _ = get_cnn_input_output(input_data,
np.zeros_like(input_data),
time_steps=time_steps)
price_data = price_data[-len(input_data):]
# split to train,test and cross validation
input_train, input_test, input_cv, price_train, price_test, price_cv = \
train_test_validation_split([input_data, price_data], split=split)
# get dims
_, input_dim, _, _ = np.shape(input_train)
# forward-propagation
x, y, logits, y_, learning_r, drop_out = cnn(input_dim,
3,
time_steps=time_steps,
filter=[1, 1])
# tf cost and optimizer
price_h = tf.placeholder(tf.float32, [None, 1])
signals = tf.constant([[1., -1., 0.]])
cost = (tf.reduce_mean(y_ * signals * price_h * 100)) # profit function
train_step = tf.train.AdamOptimizer(learning_r).minimize(-cost)
# init session
cost_hist_train, cost_hist_test, value_hist_train, value_hist_test, value_hist_cv, value_hist_train_ma, \
value_hist_test_ma, value_hist_cv_ma, step, step_hist, saving_score = [
], [], [], [], [], [], [], [], 0, [], 0.05
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess = tf.Session()
#model.compile(loss=keras.losses.categorical_crossentropy,
# optimizer=keras.optimizers.Adam(),
# metrics=['accuracy'])
class AccuracyHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('acc'))
history = AccuracyHistory()
model = models.cnn()
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_validation, y_validation),
callbacks=[history])
score = model.evaluate(x_validation, y_validation, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
plt.plot(range(1, epochs+1), history.acc)
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.show()
def train_step(train_data_gen, num_epochs, num_stock_size, num_iters):
if hp._gpu:
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
for k in range(len(physical_devices)):
tf.config.experimental.set_memory_growth(
physical_devices[k], True)
print(
'memory growth:',
tf.config.experimental.get_memory_growth(
physical_devices[k]))
else:
print("Not enough GPU hardware devices available")
M = hp.M
optimizer = tf.keras.optimizers.Adam(learning_rate=hp.lr)
if hp.model_type == "dnn":
model = models.dnn()
elif hp.model_type == "cnn":
model = models.cnn()
elif hp.model_type == "rnn":
model = models.rnn()
elif hp.model_type == "cond_dnn":
model = models.cond_dnn()
else:
print("model type error")
if hp._task == "class":
loss_fn = tf.keras.losses.BinaryCrossentropy()
else:
loss_fn = tf.keras.losses.MeanSquaredError()
#model.summary()
train_losses = []
val_losses = []
for epoch in range(num_epochs):
epoch_loss_avg = tf.keras.metrics.Mean()
val_loss_avg = tf.keras.metrics.Mean()
for i in range(num_stock_size):
data_ = next(train_data_gen) #data_[M+N]
train_data = data_[:M, :]
test_data = data_[M:, :]
for j in range(num_iters):
x_batch, y_batch = make_batch(train_data, hp.n_features)
with tf.GradientTape() as tape:
y_ = model(x_batch)
loss_value = loss_fn(y_batch, y_)
grads = tape.gradient(loss_value,
model.trainable_variables)
optimizer.apply_gradients(zip(grads,
model.trainable_variables))
epoch_loss_avg(loss_value)
num_test_iters = num_iters // 4
for j in range(num_test_iters):
x_batch, y_batch = make_batch(test_data,
hp.n_features,
_test=True)
y_ = model(x_batch)
val_loss_value = loss_fn(y_batch, y_)
val_loss_avg(val_loss_value)
train_losses.append(loss_value)
val_losses.append(val_loss_value)
print(
"Epoch {:03d}: , Number of stock time {:03d}, Loss: {:.5f}".format(
epoch, i, epoch_loss_avg.result()))
print("Val_Loss: {:.3f}".format(val_loss_avg.result()))
if hp.b_loss_plot:
losses = [train_losses, val_losses]
utils.plot_loss(losses)
return model
output_data,
time_steps=time_steps)
price_data = price_data[-len(input_data):]
# split to train, test and cross validation
input_train, input_test, input_cv, output_train, output_test, output_cv, price_train, price_test, price_cv = \
train_test_validation_split(
[input_data, output_data, price_data], split=split)
# get dims
_, input_dim, _, _ = np.shape(input_data)
_, output_dim = np.shape(output_data)
# forward-propagation
x, y, logits, y_, learning_r, drop_out = cnn(input_dim,
output_dim,
time_steps=time_steps,
filter=[3, 6])
# tf cost and optimizer
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
train_step = tf.train.AdamOptimizer(learning_r).minimize(cost)
# init session
cost_hist_train, cost_hist_test, value_hist_train, value_hist_test, value_hist_cv, value_hist_train_ma, \
value_hist_test_ma, value_hist_cv_ma, step, step_hist, saving_score = [
], [], [], [], [], [], [], [], 0, [], 0.05
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
Y = train[target_columns]
embedding_matrix, X, X_ = tp.embedded_glove_matrix(train, test,
'comment_text',
glove_path, embed_size,
max_features,
max_length)
inp = Input(shape=(max_length, ))
x = None
if model_type == 'lstm-cnn':
x = models.lstm_cnn(max_features, embed_size, embedding_matrix, inp)
elif model_type == 'lstm':
x = models.lstm(max_features, embed_size, embedding_matrix, inp)
elif model_type == 'cnn':
x = models.cnn(max_features, embed_size, embedding_matrix, inp)
model = Model(inputs=inp, outputs=x)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X, Y, batch_size=batch_size, epochs=epochs)
if save_weights:
model.save_weights('checkpoint.csv')
sub = model.predict(X_)
temp = test.copy()
temp[target_columns] = model.predict(X_)
temp.to_csv('output.csv', index=False)
client_headers = {
"content-type": "binary tensor",
"site": SITE,
"mode": MODE,
"val_type": None, # determined later in code
"step": None, # determined later in code
}
if MODE == "federated":
client_headers["val_type"] = "gradients"
elif MODE == "weightavg" or MODE == "cyclic":
client_headers["val_type"] = "weights"
#################### MODEL ####################
model = cnn(k_init, n_channels=1, n_classes=10)
model.save_weights(str(WEIGHT_DIR / "init_weights.h5"))
with open(str(MODEL_PATH), 'w') as f:
json.dump(model.to_json(), f)
opt = tf.optimizers.Adam(learning_rate=LEARNING_RATE)
# set up metrics
train_acc = tf.keras.metrics.Accuracy(name="train_acc")
train_loss = tf.keras.metrics.Mean(name="train_loss")
val_acc = tf.keras.metrics.Accuracy(name="val_acc")
val_loss = tf.keras.metrics.Mean(name="val_loss")
#################### LOAD DATA ####################
use_augm=params.use_augm)
##samp_per_epoch= params.samp_per_epoch
## validation_generator = DataGenerator(image_tr, labels_val, coords_val, index_val, params.channels,
## params.patch_size, params.batch_size, dim, shuffle=True)
# validation_generator = DataGenerator(image_tr, labels_val, coords_val, index_val, params.channels,
# params.patch_size, params.batch_size, dim,
# samp_per_epoch = params.samp_epoch_val, shuffle=True,
# use_augm = params.use_augm)
# Define optimazer
optimizer = Adam(lr=params.learning_rate)
# Define model
if params.model == "custom":
model = cnn(img_shape=dim, nb_classes=params.classes)
else:
model = DeepLabVersions(dim, params)
##plot_model(model, to_file=os.path.join(model_k,'model.png'), show_shapes=True)
cl_ind = [x for x in range(params.classes)]
losses = {
"cl_output":
categorical_focal_loss(depth=np.int(params.classes + 1),
alpha=[ratio.tolist()],
class_indexes=cl_ind)
}
#losses = {"cl_output": categorical_focal_ignoring_last_label(alpha=0.25,gamma=2)}
def get_model(input_shape):
return cnn(input_shape=input_shape)
def main():
args = parse_args()
## Create an output dir
output_dir_path = args.od + args.en
if not os.path.exists(output_dir_path):
os.makedirs(output_dir_path)
dir_name=output_dir_path
tb_dirname = output_dir_path + '/tb_logdir'
else:
counter=1
dir_name = output_dir_path
new_dir_name = dir_name
while os.path.exists(new_dir_name):
new_dir_name = dir_name + "_" + str(counter)
counter +=1
os.makedirs(new_dir_name)
dir_name=new_dir_name
tb_dirname = dir_name + "/tb_logdir"
print("===>> Output folder = {}".format(dir_name))
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.backends.cudnn.benchmark = True
torch.cuda.manual_seed(args.seed)
loaders = Cifar10Loaders()
train_loader = loaders.train_loader()
test_loader = loaders.test_loader()
if args.netqat:
base_model = cnn()
model=cnn(qat_mode=True)
## Loading checkpoints here
checkpoint = torch.load(args.load_ckpt)
##making sure that the checkpoint was loaded from disk correctly
base_model.load_state_dict(checkpoint['model_state_dict'],strict=True)
base_model_sd = base_model.state_dict()
##renaming the keys only to match the model with qat nodes. Only convs and BNs's will be changed
base_model_sd_new = map_ckpt_names(base_model_sd)
##Updating the values of keys for qat model
for k in base_model_sd_new.keys():
try:
model.state_dict()[k].copy_(base_model_sd_new[k])
print("{} successfully loaded".format(k))
except:
print("{} didnt load".format(k))
else:
model=cnn()
## Instantiate tensorboard logs
writer = SummaryWriter(tb_dirname)
images, labels = iter(train_loader).next()
img_grid = torchvision.utils.make_grid(images)
writer.add_image('cifar-10', img_grid)
#writer.add_graph(model,images)
if args.cuda:
model = model.cuda()
if args.parallel:
model = nn.DataParallel(model,device_ids=range(torch.cuda.device_count()))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)
best_test_accuracy=0
print("===>> Training started")
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch']
loss = checkpoint['loss']
print("===>>> Checkpoint loaded successfully from {} at epoch {} ".format(args.load_ckpt,epoch))
print(model)
for epoch in range(args.start_epoch, args.start_epoch + args.num_epochs):
running_loss=0.0
start=time.time()
model.train()
for i, data in enumerate(train_loader,0):
inputs, labels = data
if args.cuda:
inputs = inputs.cuda()
labels = labels.cuda()
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs,labels)
loss.backward()
optimizer.step()
running_loss +=loss.item()
if i %1000 == 999:
writer.add_scalar('train_loss',running_loss/1000, epoch * len(train_loader) + i)
writer.add_scalar('learning_rate',optimizer.param_groups[0]['lr'],epoch * len(train_loader) + i)
if epoch > 0 and epoch % args.lrdt == 0:
print("===>> decaying learning rate at epoch {}".format(epoch))
for param_group in optimizer.param_groups:
param_group['lr'] = param_group['lr'] * 0.94
running_loss /= len(train_loader)
end = time.time()
test_accuracy = calculate_accuracy(model,test_loader)
writer.add_scalar('test_accuracy',test_accuracy, epoch)
## Adding histograms after every epoch
for tag, value in model.named_parameters():
tag = tag.replace('.','/')
writer.add_histogram(tag,value.data.cpu().numpy(),epoch)
print("Epoch: {0} | Loss: {1} | Test accuracy: {2}| Time Taken (sec): {3} ".format(epoch+1, np.around(running_loss,6), test_accuracy, np.around((end-start),4)))
best_ckpt_filename = dir_name + "/ckpt_" + str(epoch) +'.pth'
##Save the best checkpoint
if test_accuracy > best_test_accuracy:
best_test_accuracy = test_accuracy
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': running_loss,
}, best_ckpt_filename)
writer.close()
print("Training finished")
y = np.loadtxt(open('data/training_labels.csv'), delimiter=',')
val = np.loadtxt(open('data/validation_data.csv'), delimiter=',')
val_y = np.loadtxt(open('data/validation_labels.csv'), delimiter=',')
#x = normalize(x)
space = PCA()
X_full = np.concatenate((X, val))
y_full = np.concatenate((y, val_y))
X_full_normal = normalize(X_full)
space.fit(X_full_normal)
X_reduced = space.transform(X_full_normal)
(real, pred) = run_nn(X_reduced, y_full)
print_results(pred, real)
X_train, X_test, y_train, y_test = train_test_split(X_reduced,
y,
test_size=0.1)
#X_res = X_train.reshape((X_train.shape[0], 12750, 1))
val_red = space.transform(val)
model = cnn()
scores = cross_val_score(model, X_reduced, y)
#model.fit(X_train, y_train, epochs=10)
#pred = model.predict(X_test)
roc_curve(pred, y_test)
np.savetxt("predictions_cnn.csv", pred)
"""
import pandas as pd
import preprocessingfile as preprocess
import models
data = 'pc2.csv'
original_data, original_X, original_Y, combined_training_data, x_train1, x_train2, x_train, x_test, x_val, y_train1, y_train2, y_train, y_test, y_val = preprocess.my_sdp_preprocessor(
data)
all_data = [
original_data, original_X, original_Y, combined_training_data, x_train1,
x_train2, x_train, x_test, x_val, y_train1, y_train2, y_train, y_test,
y_val
]
cnn_clf = models.cnn(*all_data)
svm_clf = models.svm(*all_data)
rf_clf = models.random_forest(*all_data)
nn_clf = models.NN(*all_data)
from sklearn.metrics import *
def print_accuracy(model): #nn,cnn,svm,clf
if (model == nn_clf):
y_pred_on_val = model.predict(x_val) > 0.5
y_pred_on_test = model.predict(x_test) > 0.5
elif (model == cnn_clf):
x_val_matrix = x_val.values
x_val1 = x_val_matrix.reshape(x_val_matrix.shape[0], 1,
len(x_val.columns), 1)
