def main():
parser = argparse.ArgumentParser(
description="A command line tool to manage the model")
parser.add_argument('stage',
metavar='stage',
type=str,
choices=['split', 'train', 'test', 'predict'],
help="Stage to run.")
stage = parser.parse_args().stage
if stage == 'split':
train_test_splitter = TrainTestSplitter()
train_test_splitter.split_dataset()
if stage == 'train':
print('Training model...', end="\n\n")
train_model()
if stage == 'test':
print('Testing model...', end="\n\n")
test_model()
if stage == 'predict':
print('Outputting predictions to ../outputs/predictions.csv',
end="\n\n")
predict()
def main(argv):
help_msg = '''
-g: Generate spectrograms
-s: Slice spectrograms
-c: Create and train model
-t: Test model
'''
try:
opts, args = getopt.getopt(argv, 'hgsct')
except getopt.GetoptError as err:
print(err)
print(help_msg)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(help_msg)
elif opt == '-g':
generate_spectrograms()
print('\nDone generating spectrograms')
elif opt == '-s':
slice_spectrograms()
print('\nDone slicing spectrograms')
elif opt in ('-c', '-t'):
model = create_model()
print('Done creating model')
if opt == '-c':
train_model(model)
print('Done training model')
else:
test_model(model)
print('Done testing model')
print(help_msg)
def main():
parser = argparse.ArgumentParser(description="CLI tool to manage project")
parser.add_argument('stage',
metavar='STAGE',
type=str,
choices=[
'tune', 'train', 'test', 'export', 'predict',
'algorithm_spotcheck', 'diagnose_model'
],
help='Select pipeline stage to run: %(choices)s')
stage = parser.parse_args().stage
if stage == 'tune':
tune_model(print_params=True)
elif stage == 'algorithm_spotcheck':
algorithm_spotcheck()
elif stage == 'diagnose_model':
diagnose_model()
elif stage == 'train':
train_model()
elif stage == 'test':
test_model()
elif stage == 'export':
export_model()
elif stage == 'predict':
predict()
def main():
# Creates the initial training sample and trains the model
# test_model will then make the model play games to determine how well it scores
training_data = initial_training_data()
model = train_model(training_data)
test_model(model)
# This can be put in a loop and run for x number of times
# I only did it once just to show the general idea of it
# Be careful that you don't overfit the model
secondary_data = further_training_data(model)
model = train_model(secondary_data)
test_model(model)
def test_lstm(testing_data, fer_cols):
print("Entering the Testing Phase...")
agency = rawdata.Agency.load()
for i in range(1, 97):
make_keras_picklable()
print('testing for ', i)
mod_str = 'models/model%d.h5' % i
# mod = load_model(mod_str)
mod = pickle.load(open(mod_str, 'rb'))
# mod = mod_str
print('model loaded for ', i)
# print('Fasttracking this model')
model.test_model(testing_data, mod, i, fer_cols, agency)
return agency
def main():
if len(sys.argv) != 4:
eprint("Incorrect number of arguments")
return
modelSrc = sys.argv[1]
frameWidth = int(sys.argv[2])
frameHeight = int(sys.argv[3])
instance = model.test_model(frameWidth, frameHeight)
sess = instance.load(modelSrc)
child_program.write_flag(child_program.from_child_flag.child_initialized)
while True:
flag = child_program.next_flag()
if flag == to_child_flag.test.value:
videoInput = gamepad.get_image_data(frameWidth, frameHeight)
#gamepad.display_image(videoInput)
#break
gamepadOutput = instance.run(sess, videoInput)
child_program.write_data(gamepad.compress_gamepad(gamepadOutput))
elif flag == to_child_flag.clear_lstm_state.value:
instance.clear_lstm_state(sess)
elif flag == child_program.to_child_flag.shutdown_request.value:
break
else:
eprint("Received unkown flag ", flag)
break
sess.close()
child_program.write_flag(child_program.from_child_flag.child_terminated)
def asses_param(param_name, values, n_res, train_data, test_data, base_param, test_errors, stop, ext = ""):
files = ["data/{:s}{:s}_train.npy".format(param_name, ext), "data/{:s}{:s}_test.npy".format(param_name, ext)]
np.save("data/{:s}{:s}_values.npy".format(param_name, ext), values)
if not np.all([os.path.exists(path) for path in files]):
if os.path.exists("tmp/{:s}{:s}_test.npy".format(param_name,ext)):
a = np.load("tmp/{:s}{:s}_test.npy".format(param_name,ext))
test_errors[:a.shape[0]] = a
print(param_name, "loaded")
param = base_param.copy()
n = len(values)
train_errors = np.nan*np.empty((n,n_res))
for i, v in enumerate(values):
param[param_name] = v
for j in range(n_res):
if np.isnan(test_errors[i, j]):
print("{:s}{:s} {:d}/{:d}, {:d}/{:d}".format(param_name,ext, i,n,j,n_res))
#print(param)
model = generate_model(**param)
train_errors[i,j] = train_model(model, train_data)
test_errors[i,j] = test_model(model, test_data)
#print(param_name, test_errors)
np.save("tmp/{:s}{:s}_train.npy".format(param_name,ext), train_errors[:i+1])
np.save("tmp/{:s}{:s}_test.npy".format(param_name,ext), test_errors[:i+1])
np.save("data/{:s}{:s}_train.npy".format(param_name,ext), train_errors)
np.save("data/{:s}{:s}_test.npy".format(param_name,ext), test_errors)
#print(param_name, test_errors)
else:
train_errors = np.load("data/{:s}{:s}_train.npy".format(param_name,ext))
test_errors[...] = np.load("data/{:s}{:s}_test.npy".format(param_name,ext))
#print(param_name, test_errors)
stop.value += 1
def main():
parser = argparse.ArgumentParser(
description="A command line tool to manage the project")
parser.add_argument('stage',
metavar='stage',
type=str,
choices=['tune', 'train', 'test'],
help="Stage to run.")
stage = parser.parse_args().stage
if stage == "tune":
print("Tuning model...")
tune_model()
if stage == "train":
train_model(print_params=False)
print("Model was saved")
elif stage == "test":
test_model()
def test_optimizer():
num_classes = 10
from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
img_x, img_y = 28, 28
input_shape = (img_x, img_y, 1)
X_train = X_train.reshape(X_train.shape[0], img_x, img_y, 1) / 255
X_test = X_test.reshape(X_test.shape[0], img_x, img_y, 1) / 255
y_train = to_categorical(y_train, num_classes)
y_test = to_categorical(y_test, num_classes)
model = test_model()
model.fit(X_train[0:800], y_train[0:800])
def main():
in_arg = get_input_args()
print("Command Line Arguments:\n dir =", in_arg.dir, "\n arch =",
in_arg.arch, "\n learning_rate =", in_arg.learning_rate,
"\n hidden_units =", in_arg.hidden_units, "\n epochs =",
in_arg.epochs, "\n gpu =", in_arg.gpu, "\n check_complex =",
in_arg.checkpoint_complex, "\n")
#Download and set the chosen model
model = download_model(in_arg.arch)
#Load the input data
dataloaders, datasets = load_data()
#Save class_to_idx dictionary of dataset inside the model
model.class_to_idx = datasets['train'].class_to_idx
#Recreate the classifier for use with chosen model
model = recreate_classifier(in_arg.arch, model, in_arg.hidden_units)
print(in_arg.arch.upper(), "classifier configuration:\n", model.classifier,
"\n")
#Set the checkpoint directory and name
if in_arg.checkpoint_complex:
checkpoint_name = "{}checkpoint-{}-hiddenunits{}-lr{}-epochs{}.pth".format(
in_arg.dir, in_arg.arch, in_arg.hidden_units, in_arg.learning_rate,
in_arg.epochs)
else:
checkpoint_name = "checkpoint.pth"
#Train the model and save checkpoint
model = train_model(model, dataloaders, in_arg.learning_rate,
in_arg.epochs, in_arg.gpu, checkpoint_name)
#Test the model
test_model(model, dataloaders, in_arg.gpu)
def main():
# read acc, gps, veh det for multiple drivers, scenes
X_dfs, Y_dfs = [], []
data_path = osp.join(DATA_DIR, DATA_FILE)
print(data_path)
df = read_file(data_path)
X = df.iloc[:, :-1].values.astype('float32')
labels = df.iloc[:, -1]
enc = LabelBinarizer()
Y = enc.fit_transform(labels.values)
n_classes = len(enc.classes_)
print('Number of classes:', n_classes)
print("X shape:", X.shape)
print("Y shape:", Y.shape)
scaler = MinMaxScaler(feature_range=(0, 1))
X = scaler.fit_transform(X)
seq_len, stride = 50, 1
X_seq = X_to_seq(X, seq_len, stride)
Y = Y[seq_len:]
X_tr, X_ts, Y_tr, Y_ts = train_test_split(X_seq, Y, test_size=0.2)
# train
print("X Train shape:", X_tr.shape)
print("Y Train shape:", Y_tr.shape)
print("X test shape:", X_ts.shape)
print("Y test shape:", Y_ts.shape)
n_features = X_tr.shape[-1]
train_model(X_tr, Y_tr, seq_len, n_features, n_classes)
loss = test_model(X_ts_seq, Y_ts)
print(loss)
from torchvision import models from model import test_model model_name = "resnet_101" test_model(models.resnet101,model_name)
# Testing data
n = 2500
values = np.random.uniform(-1, +1, (n, n_values))
for i in range(n_values):
values[:, i] = smoothen(values[:, i])
gates = np.random.uniform(0, 1, (n, n_gates)) < 0.01
test_data = generate_data(values, gates, last=train_data["output"][-1])
print("Training")
rmse_train = train_model(model, train_data)
print("Training error : {0:.5f}".format(rmse_train))
print("Testing")
rmse_test = test_model(model, test_data)
print("Testing error : {0:.5f}".format(rmse_test))
# Display
fig = plt.figure(figsize=(14, 8))
fig.patch.set_alpha(0.0)
n_subplots = 4
data = test_data
ax1 = plt.subplot(n_subplots, 1, 1)
ax1.patch.set_alpha(1.0)
ax1.tick_params(axis='both', which='major', labelsize=8)
ax1.plot(data["input"][:, 0], color='0.75', lw=1.0)
ax1.plot(data["output"], color='0.75', lw=1.0)
ax1.plot(model["output"], color='0.00', lw=1.5)
def main():
# read acc, gps, veh det for multiple drivers, scenes
X_dfs, Y_dfs = [], []
driver_dir = 'D1'
for drive_dir in os.listdir(osp.join(DATA_DIR, driver_dir)):
drive_path = osp.join(DATA_DIR, driver_dir, drive_dir)
print drive_path
acc = read_file(osp.join(drive_path, ACC_FILE))
gps = read_file(osp.join(drive_path, GPS_FILE))
veh = read_file(osp.join(drive_path, VEHDET_FILE))
score = read_file(osp.join(drive_path, SCORE_FILE))
datasets = [acc, gps, veh, score]
n_rows = min(map(len, datasets))
# sample high frequency data to lowest frequency
for i in range(len(datasets)):
# drop time column
datasets[i].drop(0, 1, inplace=True)
if len(datasets[i]) > n_rows:
step = len(datasets[i]) / n_rows
ndx = xrange(0, n_rows * step, step)
datasets[i] = datasets[i].ix[ndx]
datasets[i] = datasets[i].reset_index(drop=True)
score_df = datasets[-1]
datasets = datasets[:-1]
Y_df = score.ix[:, SCORE_COLUMNS]
# create dataset
X_df = pd.concat(datasets, axis=1, ignore_index=True)
X_df.fillna(0, inplace=True)
print "X:", X_df.shape
print "Y:", score_df.shape
X_dfs.append(X_df)
Y_dfs.append(Y_df)
# preprocess
X_df = pd.concat(X_dfs, ignore_index=True)
X = X_df.values.astype('float32')
Y = pd.concat(Y_dfs, ignore_index=True).values
print "X shape:", X.shape
print "Y shape:", Y.shape
scaler = MinMaxScaler(feature_range=(0, 1))
X = scaler.fit_transform(X)
X_tr, X_ts, Y_tr, Y_ts = train_test_split(X, Y, test_size=0.2)
# train
print "X Train shape:", X_tr.shape
print "Y Train shape:", Y_tr.shape
print "X test shape:", X_ts.shape
print "Y test shape:", Y_ts.shape
seq_len = 16
X_tr_seq = X_to_seq(X, seq_len, 1)
Y_tr = Y_tr[seq_len:]
X_ts_seq = X_to_seq(X_ts, seq_len, 1)
Y_ts = Y_ts[seq_len:]
#train_model(X_tr, Y_tr)
loss = test_model(X_ts_seq, Y_ts)
print loss
from torchvision import models from model import test_model model_name = "vgg_19" test_model(models.vgg19, model_name)
from data_handler import prepare_dataset
from model import train_model
from model import test_model
dataset = prepare_dataset()
print("Dataset Prepared")
model = train_model(dataset)
print("Model Trained")
test_model(model)
print("Model Tested")
def main(cfg):
"""
# 訓練データと検証データのパス
train_list = make_datapath_list(
csv_file=cfg.csv.train, data_id=cfg.csv.id, data_dir=cfg.data.train_dir
)
val_list = make_datapath_list(
csv_file=cfg.csv.val, data_id=cfg.csv.id, data_dir=cfg.data.val_dir
)
test_list = make_datapath_list(
csv_file=cfg.csv.test, data_id=cfg.csv.id, data_dir=cfg.data.test_dir
)
# 画像表示と確認
size = 224
mean = (0.485, 0.456, 0.406)
std = (0.229, 0.224, 0.225)
img = train_list[100]
img = Image.open(img)
plt.imshow(img)
plt.show()
transform = ImageTransform(size, mean, std)
img_transformed = transform(img)
img_transformed = img_transformed.numpy().transpose((1, 2, 0))
img_transformed = np.clip(img_transformed, 0, 1)
plt.imshow(img_transformed)
plt.show()
# データセットの作成
train_dataset = IsicDataset(
file_list=train_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="train",
csv_file=cfg.csv.train,
label_name=cfg.csv.label,
)
val_dataset = IsicDataset(
file_list=val_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="val",
csv_file=cfg.csv.val,
label_name=cfg.csv.label,
)
test_dataset = IsicDataset(
file_list=test_list,
transform=ImageTransform(cfg.image.size, cfg.image.mean, cfg.image.std),
phase="test",
csv_file=cfg.csv.test,
label_name=cfg.csv.label,
)
"""
# Imagedatafolderを用いたデータセットの作成(使わない場合はコメントアウト)
# 学習用データセット作成
train_dataset = make_trainset(
dataroot=cfg.data.train_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 検証用データセット
val_dataset = make_testset(
dataroot=cfg.data.val_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 検証用データセット
test_dataset = make_testset(
dataroot=cfg.data.test_dir,
resize=cfg.image.size,
mean=cfg.image.mean,
std=cfg.image.std,
)
# 辞書型'train'と'val'と'test'のデータローダを作成
dataloaders_dict = create_dataloader(
batch_size=cfg.image.batch_size,
train_dataset=train_dataset,
val_dataset=val_dataset,
test_dataset=test_dataset,
)
# バッチごとの動作確認
batch_iterator = iter(dataloaders_dict["train"])
inputs, labels = next(batch_iterator)
print(inputs.size())
print(labels)
# ネットワークモデルのロード
net = models.vgg16_bn(pretrained=True)
log.info(net)
net.classifier[6] = nn.Linear(in_features=4096, out_features=2)
net.classifier[2] = nn.Dropout(p=0.6)
net.classifier[5] = nn.Dropout(p=0.6)
net.train()
# 損失関数の設定
criterion = nn.CrossEntropyLoss()
# criterion = nn.BCELoss()
log.info(net)
# 調整するパラメータの設定
params_to_update = []
update_param_names = cfg.train.update_param_names
# update_param_namesに含まれているパラメータだけ調整
for name, param in net.named_parameters():
if name in update_param_names:
param.requires_grad = True
params_to_update.append(param)
log.info(name)
else:
param.requires_grad = False
# 調整するパラメータ名をログに保存
log.info(params_to_update)
# 最適化手法の設定
optimizer = optim.SGD(params=params_to_update,
lr=cfg.optimizer.lr,
momentum=cfg.optimizer.momentum)
log.info(optimizer)
# 学習回数を設定ファイルから読み込む
num_epochs = cfg.train.num_epochs
# GPU初期設定
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print("使用デバイス: ", device)
# ネットワークをGPUへ
net.to(device)
torch.backends.cudnn.benchmark = True
# 損失値と認識率を保持するリスト
train_loss = []
train_acc = []
test_loss = []
test_acc = []
# 学習と検証
for epoch in range(num_epochs):
log.info("Epoch {} / {}".format(epoch + 1, num_epochs))
log.info("----------")
# 学習
train_history = train_model(net, dataloaders_dict["train"], criterion,
optimizer)
# 学習したlossと認識率のリストを作成
train_loss.append(train_history["train_loss"])
train_acc.append(train_history["train_acc"])
# 検証
test_history = test_model(net, dataloaders_dict["val"], criterion)
# 検証したlossと認識率のリストを作成
test_loss.append(test_history["test_loss"])
test_acc.append(test_history["test_acc"])
# figインスタンスとaxインスタンスを作成
fig_loss, ax_loss = plt.subplots(figsize=(10, 10))
ax_loss.plot(range(1, num_epochs + 1, 1), train_loss, label="train_loss")
ax_loss.plot(range(1, num_epochs + 1, 1), test_loss, label="test_loss")
ax_loss.set_xlabel("epoch")
ax_loss.legend()
fig_loss.savefig("loss.png")
fig_acc, ax_acc = plt.subplots(figsize=(10, 10))
ax_acc.plot(range(1, num_epochs + 1, 1), train_acc, label="train_acc")
ax_acc.plot(range(1, num_epochs + 1, 1), test_acc, label="test_acc")
ax_acc.legend()
ax_acc.set_xlabel("epoch")
fig_acc.savefig("acc.png")
"""
# Pytorchのネットワークパラメータのロード
# 現在のディレクトリを取得
current_dir = pathlib.Path(__file__).resolve().parent
print(current_dir)
# 学習済みのパラメータを使用したいとき
load_path = str(current_dir) + "/weights_fine_tuning.pth"
load_weights = torch.load(load_path)
net.load_state_dict(load_weights)
"""
evaluate_history = evaluate_model(net, dataloaders_dict["test"], criterion)
print(evaluate_history["confusion_matrix"])
# 性能評価指標の計算（正解率、適合率、再現率、F1値)
efficienct = calculate_efficiency(evaluate_history["confusion_matrix"])
log.info("正解率: " + str(efficienct["accuracy"]))
log.info("適合率: " + str(efficienct["precision"]))
log.info("再現率: " + str(efficienct["recall"]))
log.info("f1値 :" + str(efficienct["f1"]))
# 混同行列の作成と表示
fig_conf, ax_conf = plt.subplots(figsize=(10, 10))
sns.heatmap(
evaluate_history["confusion_matrix"],
annot=True,
fmt="d",
cmap="Reds",
)
ax_conf.set_title("confusion_matrix")
ax_conf.set_xlabel("Predicted label")
ax_conf.set_ylabel("True label")
fig_conf.savefig("confusion_matrix.png")
# パラメータの保存
save_path = "./melanoma_nevi_classifier.pth"
torch.save(net.state_dict(), save_path)
values = np.random.uniform(-1, +1, n)
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.01
train_data = generate_data(values, ticks)
# Testing data
n = 2500
values = smoothen(np.random.uniform(-1, +1, n))
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.01
test_data = generate_data(values, ticks, last=train_data["output"][-1])
# Train model
error = train_model(model, train_data)
print("Training error : {0}".format(error))
# Test model and collect internal states
error = test_model(model, test_data)
print("Testing error : {0}".format(error))
testing_states = model["state"].copy()
# Run model on training data to collect internal states
error = test_model(model, train_data)
print("Training error : {0}".format(error))
training_states = model["state"].copy()
# Find reservoir units correlation / output on training data
from scipy.stats.stats import pearsonr
n = len(model["state"])
C = np.zeros(n)
for i in range(n):
C[i], p = pearsonr(model["state"][i].ravel(),
model["output"].ravel())
n = 25000
values = np.random.uniform(-1, +1, n)
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.01
train_data = generate_data(values, ticks)
error = train_model(model, train_data)
print("Training error : {0}".format(error))
# Testing data
n = 2500
values = np.cos(np.linspace(0, 20 * np.pi, n))
ticks = np.zeros(n)
ticks[::25] = 1
test_data = generate_data(values, ticks, last=train_data["output"][-1])
error = test_model(model, test_data)
print("Testing error : {0}".format(error))
# Display
fig = plt.figure(figsize=(14, 8))
fig.patch.set_alpha(0.0)
n_subplots = 4
data = test_data
ax1 = plt.subplot(n_subplots, 1, 1)
ax1.tick_params(axis='both', which='major', labelsize=8)
ax1.plot(data["input"][:, 0], color='0.75', lw=1.0)
ax1.plot(data["output"], color='0.75', lw=1.0)
ax1.plot(model["output"], color='0.00', lw=1.5)
# ('all->pol2 (train on all and test on poles (with no negs))', pol_pos_test, empty_test),
('all->pol3 (train on all and test on poles)', pol_pos_test,
pol_neg_test)
])
]
# for (name, (train_pos, train_neg), (test_pos, test_neg)) in eval_pairs:
for train_tuple, test_tuples in eval_pairs:
name, epochs, train_pos, train_neg = train_tuple
log('Exp: ' + name + ', epochs: ' + str(epochs))
X, y = create_dataset(train_pos, train_neg)
X, y = shuffle(X, y)
train_split = int(0.8 * X.shape[0])
train_set = get_neon_set(X[:train_split], y[:train_split])
val_set = get_neon_set(X[train_split:], y[train_split:])
model = fit_model(train_set, val_set, num_epochs=epochs)
train_error = test_model(model, train_set)
log('Train Misclassification error = %.2f%%' % train_error)
val_error = test_model(model, val_set)
log('Val Misclassification error = %.2f%%' % val_error)
for test_tuple in test_tuples:
name, test_pos, test_neg = test_tuple
log('Exp: ' + name)
X_test, y_test = create_dataset(test_pos, test_neg)
test_set = get_neon_set(X_test, y_test)
test_error = test_model(model, test_set)
log(' Test Misclassification error = %.2f%%' % test_error)
log('')
model.get_description()
model.save_params('eq_polar_params.p')
# Training data
n = 25000 # 300000
values = np.random.uniform(-1, +1, n)
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.01
train_data = generate_data(values, ticks)
# Testing data
n = 2500
values = smoothen(np.random.uniform(-1, +1, n))
ticks = np.random.uniform(0, 1, (n, n_gate)) < 0.01
test_data = generate_data(values, ticks, last=train_data["output"][-1])
error = train_model(model, train_data)
print("Training error : {0}".format(error))
error = test_model(model, test_data)
print("Testing error : {0}".format(error))
np.save(files[0], test_data)
np.save(files[1], model["output"])
np.save(files[2], model["state"])
else:
test_data = np.load(files[0])
model = {}
model["output"] = np.load(files[1])
model["state"] = np.load(files[2])
error = np.sqrt(np.mean((model["output"] - test_data["output"])**2))
print("Testing error : {0}".format(error))
# Display
data = test_data
if sys.argv[1] == "preprocess":
# Preprocess routine
from preprocess import preprocess_file
preprocess_file(args_dict['input'], args_dict['output'])
elif sys.argv[1] == "encode_features":
# Build dict to encode features into labels
from feature_encoder import build_categorical_feature_dict
build_categorical_feature_dict(args_dict['input'], args_dict['output'])
elif sys.argv[1] == "train_model":
# Model training routine
from model import train_model
train_model(args_dict['input'], args_dict['encode_feature_dict'],
args_dict['training_set_output'],
args_dict['testing_set_output'],
args_dict['classifier_model_output'],
args_dict['regressor_model_output'])
elif sys.argv[1] == "predict":
# Model prediction routine
# Testing/foreign set should be used here
from model import test_model
test_model(args_dict['input'], args_dict['encode_feature_dict'],
args_dict['classifier_model'], args_dict['regressor_model'],
args_dict['result_output'])
model.to(device)
# loss function
loss_fn = nn.BCEWithLogitsLoss(reduction='sum')
# optimizer
optimizer = torch.optim.SGD(model.parameters(), lr=config['lr'], momentum=0.9)
# scheduler
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
config['scheduler_stepsize'],
eta_min=0,
last_epoch=-1)
# train the model
# model = train_model(model, dataloaders, loss_fn, optimizer, config['maxnumepochs'], scheduler, device, class_num)
# save the model
# torch.save(obj = model.state_dict(), f = "./models/model.pth")
# load the model
model.load_state_dict(torch.load('./models/model.pth', map_location='cpu'))
# test the model
_ = test_model(model,
dataloaders['val'],
loss_fn,
device,
20,
saveResult=False)
def model_currency(k=10):
"""
Creates a neural network to model a currency.
:param k: The degree of cross-validation to be performed.
:return:
"""
coin_dict, data = cr.read_csv()
data = cr.split_data_coins(coin_dict, data)
coin = select_currency(data)
data = data[coin]
model_weights = []
model_errors = []
split_data = cr.split_data(data, k)
split_data = [[[float(e[2]), float(e[5]), float(e[3]), float(e[4])] for e in s] for s in split_data]
print("Modeling neural networks with k-fold cross-validation")
for i in range(k):
model = m.create_model(4, [8, 8, 2])
raw_data = split_data[:i] + split_data[i+1:]
training_data = np.array([s[:-1] for s in raw_data])
m.train_model(model, training_data, np.array([to_expected(s) for s in raw_data]))
error = m.test_model(model, np.array([split_data[i][:-1]]), np.array([to_expected(split_data[i])]))
model_weights.append(np.array(m.get_weights(model)))
model_errors.append(error[0])
sum_error = sum(1/e for e in model_errors)
for idx, error in enumerate(model_errors):
proportion = (1/error)/sum_error
model_weights[idx] = proportion * model_weights[idx]
true_weights = sum(model_weights)
true_model = m.create_model(4, [8, 8, 2])
m.set_weights(true_model, true_weights)
while True:
print("For how long would you like to invest?")
steps = input("Choice: ")
try:
steps = int(steps)
assert steps > 0
except ValueError or AssertionError:
print("That was not a valid amount of time.")
break
revenue = m.predict_model(true_model, np.array([[split_data[-1][-1]]]), steps)
error = m.test_model(true_model, np.array([s[:-1] for s in split_data]), np.array([to_expected(s) for s in split_data]))
multiply = [1, 1]
for r in revenue:
multiply[0] *= r[0][0]
multiply[1] *= r[0][1]
print("Expected revenue: {} with error percentage at: {}%".format(multiply, error[0]*100))
return revenue, error
image_filepath = args.image_filepath
topk = args.topk
#select device
device = set_device(device_selection)
# load model
loaded_model, criterion, optimizer, checkpoint = load_checkpoint(
checkpoint_filename, pretrained_model_selection, learning_rate, device)
# Extract and Transform data
train_data, valid_data, test_data, trainloader, testloader, validloader = load_and_transform_data(
data_dir)
# check device
print("Is our device GPU?")
print(device == torch.device("cuda"))
# test the model but only if it's GPU, on CPU it'll run forever. Purpose: to see if the model is fine after saving a checkpoint and loading it
if device == torch.device("cuda"):
test_model(testloader, device, loaded_model)
else:
pass
# label mapping
cat_to_name = load_cat_to_name(cat_to_name_filename)
# predict
probs_top_list, classes_top_list = predict_one_image(image_filepath,
loaded_model, topk,
cat_to_name)
from torchvision import models from model import test_model model_name = "resnet_50" test_model(models.resnet50, model_name)
if not model:
model = create_model(args.type,
x_train.shape,
args.dense_nodes,
filters=args.filters)
model_name = save_model(None)
if epochs[0] > 0:
(model, history) = train_model(model, x_train, y_train,
epochs[0], 'adadelta',
args.batch_size)
data_obj_test = create_data(args.dataset, args.component)
(x_test, y_test) = format_data(data_obj_test, args.timesteps)
test_model(model, x_test, y_test, data_obj_test, history,
args.batch_size, model_name, 'Adadelta',
component_name, args.plot)
if epochs[1] > 0:
(model, history) = train_model(model, x_train, y_train,
epochs[1], 'adam',
args.batch_size)
save_model(model, model_name)
data_obj_test = create_data(args.dataset, args.component)
(x_test, y_test) = format_data(data_obj_test, args.timesteps)
test_model(model, x_test, y_test, data_obj_test, history,
args.batch_size, model_name, 'Adam', component_name,
args.plot)
def train_model(params, emb_weight, train_loader, val_loader, test_loader,
device):
tags_predicted = params['tags_predicted']
num_tasks = len(tags_predicted)
rnn1_type = params['rnn1_type']
rnn_1 = rnn_types[rnn1_type]
rnn2_type = params['rnn2_type']
rnn_2 = rnn_types[rnn2_type]
bi = params['bi']
hidden_dim1 = params['hidden_dim1']
hidden_dim2 = params['hidden_dim2']
multi_task_train = params['multi_task_train']
num_classes = params['num_classes']
batch_size = params['batch_size']
cuda_on = params['cuda_on']
weights_matrix = torch.from_numpy(emb_weight)
model = two_stage_RNN(rnn_1, hidden_dim1, bi, rnn_2, hidden_dim2,
batch_size, cuda_on, weights_matrix, num_tasks,
num_classes)
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
print('The number of train parameters',
sum([np.prod(p.size()) for p in model_parameters]))
model = model.to(device)
#parameter for training
learning_rate = params['learning_rate']
num_epochs = params['num_epochs'] # number epoch to train
# Criterion and Optimizer
#pos_weight=torch.Tensor([40,]).cuda()
criterion = nn.BCEWithLogitsLoss(
pos_weight=loss_weight
) #torch.nn.BCELoss(); torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
train_loss_list = []
train_AUC_list = []
train_ACC_list = []
val_AUC_list = []
val_ACC_list = []
max_val_auc = 0
step_max_descent = params['step_max_descent']
for epoch in range(num_epochs):
for i, (steps_batch, lengths_batch,
labels_batch) in enumerate(train_loader):
for step_id in range(6):
lengths_batch[step_id] = lengths_batch[step_id].to(device)
steps_batch[step_id] = steps_batch[step_id].to(device)
model.train()
optimizer.zero_grad()
logits = model(steps_batch, lengths_batch)
if num_tasks == 1:
task_id = 0
else:
print('Task number is greater than 1')
loss = criterion(
logits[task_id],
labels_batch[task_id].view(-1, 1).float().to(device))
train_loss_list.append(loss.item())
loss.backward()
optimizer.step()
# validate every 10 iterations
if i % 10 == 0:
val_auc, val_acc = test_model(val_loader, model)
val_ACC_list.append(val_acc[task_id])
val_AUC_list.append(val_auc[task_id])
print(
'{}/{}, Step:{}/{}, TrainLoss:{:.6f}, ValAUC:{} ValAcc:{}'.
format(epoch + 1, num_epochs, i + 1, len(train_loader),
loss, val_auc, val_acc))
# train_auc, train_acc = test_model(train_loader, model)
# train_AUC_list.append(train_auc)
# train_ACC_list.append(train_acc)
# early stop
if max_val_auc < val_auc[task_id]:
max_val_auc = val_auc[task_id]
step_num_descent = 0
else:
step_num_descent += 1
if step_max_descent == step_num_descent:
print('early stop!')
break
val_auc, val_acc = test_model(val_loader, model)
train_auc, train_acc = test_model(train_loader, model)
print('Epoch: [{}/{}], trainAUC: {}, trainAcc: {}'.format(
epoch + 1, num_epochs, train_auc, train_acc))
print('Epoch: [{}/{}], ValAUC: {}, ValAcc: {}'.format(
epoch + 1, num_epochs, val_auc, val_acc))
if step_max_descent == step_num_descent:
break
val_auc_mean = np.mean(val_AUC_list[-step_max_descent * 2 - 1:])
val_acc_mean = np.mean(val_ACC_list[-step_max_descent * 2 - 1:])
return val_auc_mean, val_acc_mean, model
def train_model(params, emb_weight, train_loader, val_loader, test_loader, loss_weights):
rnn1_type = params['rnn1_type']
rnn_1 = rnn_types[rnn1_type]
rnn2_type = params['rnn2_type']
rnn_2 = rnn_types[rnn2_type]
bi = params['bi']
tags_predicted = params['tags_predicted']
num_tasks = len(tags_predicted)
hidden_dim1 = params['hidden_dim1']
hidden_dim2 = params['hidden_dim2']
multi_task_train = params['multi_task_train']
num_classes = params['num_classes']
batch_size = params['batch_size']
cuda_on = params['cuda_on']
weights_matrix = torch.from_numpy(emb_weight)
model = two_stage_RNN(rnn_1, hidden_dim1, bi, rnn_2, hidden_dim2, batch_size,
cuda_on, weights_matrix, num_tasks, num_classes)
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
print('The number of train parameters', sum([np.prod(p.size()) for p in model_parameters]))
model = model.to(device)
#parameter for training
learning_rate = params['learning_rate']
num_epochs = params['num_epochs'] # number epoch to train
# Criterion and Optimizer
#pos_weight=torch.Tensor([40,]).cuda()
criterion = {}
if loss_weights is None:
for i in range(num_tasks):
criterion[i] = nn.BCEWithLogitsLoss() #torch.nn.BCELoss(); torch.nn.CrossEntropyLoss()
else:
for i in range(num_tasks):
criterion[i] = nn.BCEWithLogitsLoss(pos_weight=loss_weights[i])
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
train_loss_list = []
# train_AUC_list = defaultdict(float)
# train_ACC_list = defaultdict(float)
val_AUC_dict = defaultdict(list)
val_ACC_dict= defaultdict(list)
max_val_auc = defaultdict(float)
step_max_descent = params['step_max_descent']
val_auc_mean = defaultdict(float)
val_acc_mean = defaultdict(float)
for epoch in range(num_epochs):
for i, (steps_batch, lengths_batch, labels_batch) in enumerate(train_loader):
for step_id in range(6):
lengths_batch[step_id] = lengths_batch[step_id].to(device)
steps_batch[step_id] = steps_batch[step_id].to(device)
model.train()
optimizer.zero_grad()
logits = model(steps_batch, lengths_batch)
if multi_task_train == 'mean_loss':
loss_list = []
for task_id in range(num_tasks):
loss_list.append(criterion[task_id](logits[task_id],
labels_batch[task_id].view(-1,1).float().to(device)))
loss= torch.mean(torch.stack(loss_list))
elif multi_task_train == 'random_selection':
task_id = np.random.randint(0, num_tasks)
loss = criterion[task_id](logits[task_id], labels_batch[task_id].view(-1,1).float().to(device))
else:
print('multi-task-train-method Error')
train_loss_list.append(loss.item())
loss.backward()
optimizer.step()
# validate every 100 iterations
if i % 10 == 0:
val_auc, val_acc = test_model(val_loader, model)
for key in val_auc.keys():
val_ACC_dict[key].append(val_acc[key])
val_AUC_dict[key].append(val_auc[key])
print('{}/{}, Step:{}/{}, TrainLoss:{:.6f}, ValAUC:{} ValAcc:{}'.format(
epoch+1, num_epochs, i+1, len(train_loader), loss, val_auc, val_acc))
# train_auc, train_acc = test_model(train_loader, model)
# train_AUC_list.append(train_auc)
# train_ACC_list.append(train_acc)
# early stop
flag_increase = False
for key in val_auc.keys():
if max_val_auc[key] < val_auc[key]:
max_val_auc[key] = val_auc[key]
flag_increase = True
if flag_increase == True:
step_num_descent = 0
else:
step_num_descent += 1
if step_max_descent == step_num_descent:
print('early stop!')
break
val_auc, val_acc = test_model(val_loader, model)
train_auc, train_acc = test_model(train_loader, model)
print('Epoch: [{}/{}], trainAUC: {}, trainAcc: {}'.format(epoch+1, num_epochs, train_auc.values(), train_acc.values()))
print('Epoch: [{}/{}], ValAUC: {}, ValAcc: {}'.format(epoch+1, num_epochs, val_auc.values(), val_acc.values()))
if step_max_descent == step_num_descent:
break
for key in val_AUC_dict.keys():
val_auc_mean[key] = np.mean(val_AUC_dict[key][-step_max_descent*2-1:])
val_acc_mean[key] = np.mean(val_ACC_dict[key][-step_max_descent*2-1:])
return val_auc_mean, val_acc_mean, model
def main():
# parse the raw data files first
normal_file_raw = 'dataset/normalTrafficTraining.txt'
anomaly_file_raw = 'dataset/anomalousTrafficTest.txt'
normal_test_raw = 'dataset/normalTrafficTest.txt'
normal_test_parse = 'dataset/normalRequestTest.txt'
normal_file_parse = 'dataset/normalRequestTraining.txt'
anomaly_file_parse = 'dataset/anomalousRequestTest.txt'
# Parse the files to decode the URLs in the raw HTTP requests and write them in a proper format
parse_file(normal_file_raw, normal_file_parse)
parse_file(anomaly_file_raw, anomaly_file_parse)
parse_file(normal_test_raw, normal_test_parse)
# Convert each HTTP request into a string and append each of these strings to a list
X_train = to_string('../input/normalRequestTraining.txt')
X_test_bad = to_string('../input/anomalousRequestTest.txt')
X_test_good = to_string('../input/normalRequestTest.txt')
# Label the good requests and bad requests
# 0 --> good --> [1. 0.]
# 1 --> bad --> [0. 1.]
y_train = [0] * len(X_train)
y_bad = [1] * len(X_test_bad)
y_good = [0] * len(X_test_good)
# Put all the requests in the X and y lists
y_unshuffled = y_bad + y_good + y_train
X_unshuffled = X_test_bad + X_test_good + X_train
# Shuffle the data
X_shuffled, y_shuffled = shuffle(X_unshuffled, y_unshuffled)
# use categorical output
y_shuffled = to_categorical(y_shuffled)
# set parameters:
subset = None
# Maximum length. Longer gets chopped. Shorter gets padded.
maxlen = 1000
# Model params
# Filters for conv layers
nb_filter = 64
# Number of units in the dense layer
dense_outputs = 64
# Conv layer kernel size
filter_kernels = [7, 7]
# Number of units in the final output layer. Number of classes.
cat_output = 2
# Compile/fit params
batch_size = 128
nb_epoch = 20
print('Loading data...')
# # Expect x to be a list of sentences. Y to be index of the categories.
(xt, yt), (x_test, y_test) = load_data(X_shuffled, y_shuffled)
print('Creating vocab...')
vocab, reverse_vocab, vocab_size, alphabet = create_vocab_set()
print('Compile model...')
model = create_model(filter_kernels, dense_outputs, maxlen, vocab_size,
nb_filter, cat_output)
# Encode data
xt = encode_data(xt, maxlen, vocab)
x_test = encode_data(x_test, maxlen, vocab)
print('Chars vocab: {}'.format(alphabet))
print('Chars vocab size: {}'.format(vocab_size))
print('X_train.shape: {}'.format(xt.shape))
model.summary()
print('Fit model...')
patience = 5 # this is the number of epochs with no improvment after which the training will stop
history = fit_model(model, xt, yt, patience, batch_size, nb_epoch)
print("Testing model...")
score = test_model(x_test, y_test, batch_size)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
# Graphs and data visualisation
# Training Accuracy Vs validation Accuracy
plt.figure(0)
plt.figsize = (10, 10)
plt.plot(history.history['acc'], 'r')
plt.plot(history.history['val_acc'], 'g')
plt.xticks(np.arange(0, 20, 1.0))
plt.xlabel("Num of Epochs")
plt.ylabel("Accuracy")
plt.title("Training Accuracy Vs validation Accuracy")
plt.legend(['train', 'validation'])
# Training Loss Vs Validation Loss
plt.figure(0)
plt.figsize = (10, 10)
plt.plot(history.history['loss'], 'r')
plt.plot(history.history['val_loss'], 'g')
plt.xticks(np.arange(0, 20, 1.0))
plt.yticks(np.arange(0, 0.5, 0.1))
plt.xlabel("Num of Epochs")
plt.ylabel("Loss")
plt.title("Training Loss Vs validation Loss")
plt.legend(['train', 'validation'])
# Classification Matrix
y_pred = model.predict(x_test)
y_pred1 = (y_pred > 0.5)
matrix = confusion_matrix(y_test.argmax(axis=1), y_pred1.argmax(axis=1))
print(matrix)
plt.matshow(matrix, cmap=plt.cm.gray)
plt.show()
row_sum = matrix.sum(axis=1, keepdims=True)
norm_conf = matrix / row_sum
print(norm_conf)
plt.matshow(norm_conf, cmap=plt.cm.gray)
plt.show()
