def main():
x, y, name = data_manager.preprocess_phoneme(DATASET_DIR, TRAIN_RATIO, DATA_LENGTH, DATA_STRIDE, BATCH_SIZE, NORMALIZE)
print('Data Loaded')
acc = data_manager.Data(np.zeros(EPOCH), None, np.zeros(EPOCH))
loss = data_manager.Data(np.zeros(EPOCH), None, np.zeros(EPOCH))
model = model_archive.CONV1D2_256_128_64_32(x.train.shape[2], NUM_CLASS)
wrapper = Wrapper(model, LEARN_RATE)
print('\n--------- Training Start ---------')
for epoch in range(EPOCH):
_, acc.train[epoch], loss.train[epoch] = wrapper.run_model(x.train, y.train, DEVICE, 'train')
_, acc.valid[epoch], loss.valid[epoch] = wrapper.run_model(x.valid, y.valid, DEVICE, 'eval')
if wrapper.early_stop(loss.valid[epoch]): break
print('Epoch [' + str(epoch+1).zfill(int(np.log10(EPOCH))+1) + '/' + str(EPOCH) + ']'
+ ' acc : ' + str(round(acc.train[epoch],4)) + ' - val_acc : ' + str(round(acc.valid[epoch],4))
+ ' | loss : ' + str(round(loss.train[epoch],4)) + ' - val_loss : ' + str(round(loss.valid[epoch],4)))
print('-------- Training Finished -------')
pred_test, acc_test, _ = wrapper.run_model(x.test, y.test, DEVICE, 'eval')
accuracy, precision, recall, fscore = data_manager.evaluate(pred_test, y.test)
print('\nClassification Accuracy : ' + str(round(acc_test,4)))
print('Binary Accuracy : ' + str(round(accuracy,4)))
print('Precision : ' + str(round(precision,4)))
print('Recall : ' + str(round(recall,4)))
print('F SCORE : ' + str(round(fscore,4)))
wrapper.export(EXPORT_DIR, x.test, y.test, pred_test)
print('\nFiles exported to ' + os.path.abspath(EXPORT_DIR))
def test_add_events_broken(self):
"""
Test that the broken events are safely detected.
"""
d = dm.Data()
prep_data(d)
event_list = glob.glob(test_data_broken + "trace.*.json")
event_list.sort(key=lambda x: int(x.split('.')[-2]))
for i in range(len(event_list)):
info = []
with open(event_list[i], 'r') as f:
info = json.load(f)
d.set_FOI(info['value']['foi'])
d.set_labels(info['value']['labels'])
self.assertRaises(Exception, d.add_events, info['value']['events'])
break
def test_generate_forest_after_cleaned(self):
"""
Test that the forest is not generated safely.
"""
d = dm.Data()
prep_data(d)
event_list = glob.glob(test_data + "trace.*.json")
event_list.sort(key=lambda x: int(x.split('.')[-2]))
for i in range(len(event_list)):
info = []
with open(event_list[i], 'r') as f:
info = json.load(f)
d.set_FOI(info['value']['foi'])
d.set_labels(info['value']['labels'])
d.add_events(info['value']['events'])
d.generate_forest()
self.assertEqual(len(d.forest), 0)
def test_add_events(self):
"""
Test that the new events are correctly parsed.
"""
d = dm.Data()
prep_data(d)
event_list = glob.glob(test_data + "trace.*.json")
event_list.sort(key=lambda x: int(x.split('.')[-2]))
for i in range(len(event_list)):
info = []
with open(event_list[i], 'r') as f:
info = json.load(f)
d.set_FOI(info['value']['foi'])
d.set_labels(info['value']['labels'])
d.add_events(info['value']['events'])
i = 0
ans = [177367, 216175]
for k in d.events.keys():
self.assertEqual(len(d.events[k]), ans[i])
i += 1
def main():
args = parse_args()
print("Params:")
print(args)
print()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
X = tf.placeholder(tf.float32, [17770, None], name='X')
Y = tf.placeholder(tf.float32, [17770, None], name='Y')
Yhat, weights = model.autoencoder(X,
args.layers,
keep_prob=(1.0 - args.dropout),
constrained=args.constrained)
YhatDev, weights = model.autoencoder(X,
args.layers,
constrained=args.constrained,
weights=weights)
loss = model.get_loss(Y, Yhat)
loss_sum, loss_examples = model.get_test_loss(Y, Yhat)
loss_sum_dev, loss_examples_dev = model.get_test_loss(Y, YhatDev)
losses = (loss, loss_sum, loss_examples, loss_sum_dev, loss_examples_dev)
optimizer = model.get_optimizer(args.optimizer_type, args.lr,
args.momentum)
if args.small_dataset:
train_path = "../data/netflix/output_small_train"
dev_path = "../data/netflix/output_small_dev"
test_path = "../data/netflix/output_small_test"
else:
train_path = "../data/netflix/output_train"
dev_path = "../data/netflix/output_dev"
test_path = "../data/netflix/output_test"
data_train = data_manager.Data(size=args.chunk_size,
batch=args.batch_size,
path=train_path)
data_dev = data_manager.Data(size=args.chunk_size,
batch=args.batch_size,
path=dev_path,
test=True)
data_test = data_manager.Data(size=args.chunk_size,
batch=args.batch_size,
path=test_path,
test=True)
train_losses, eval_losses = model.train(
data_train,
data_dev,
losses,
optimizer,
X,
Y,
Yhat,
epochs=args.epochs,
dense_refeeding=args.dense_refeeding)
model.test(data_test, X, Y, YhatDev)
t, = plt.plot([i + 1 for i in range(len(train_losses))],
train_losses,
label="Train")
e, = plt.plot([i + 1 for i in range(len(eval_losses))],
eval_losses,
label="Dev")
plt.legend(handles=[t, e])
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.show()
print([i + 1 for i in range(len(train_losses))])
print(train_losses)
print([i + 1 for i in range(len(eval_losses))])
print(eval_losses)