def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# create an instance of the model you want
model = MyModel(config)
# create your data generator
data = DataGenerator(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = SimpleTrainer(sess, model, data, config, logger)
saverExternal = tf.train.Saver(var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='external'))
saverExternal.restore(sess, "experiments/model_8/model8.ckpt_2")
# here you train your model
trainer.train()
trainer.validate()
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# create an instance of the model you want
model = MyModel(config)
# create your data generator
data = DataGenerator(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = SimpleTrainer(sess, model, data, config, logger)
# here you train your model
trainer.train()
trainer.validate()
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config.tensorboard_log_dir, config.checkpoint_dir, "val_test"])
print('Create the data generator.')
if hasattr(config, "data_set"):
if config.data_set == "face_data_77":
data_loader = FaceLandmark77DataLoader(config)
else:
data_loader = SimpleMnistDataLoader(config)
else:
data_loader = SimpleMnistDataLoader(config)
print('Create the model.')
if hasattr(config, "model_name"):
if config.model_name == "mobile_net":
model = MobileNetV2Model(config)
else:
model = SimpleMnistModel(config)
else:
model = SimpleMnistModel(config)
print(model.model.input_names)
print([out.op.name for out in model.model.outputs])
return
if hasattr(config, "best_checkpoint"):
model.load(config.best_checkpoint)
frozen_graph = freeze_session(K.get_session(),
output_names=[out.op.name for out in model.model.outputs])
ckpt_path = Path(config.best_checkpoint)
tf.train.write_graph(frozen_graph, str(ckpt_path.parent), ckpt_path.with_suffix(".pb").name, as_text=False)
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])
print('Create the data generator.')
if hasattr(config, "data_set"):
if config.data_set == "face_data_77":
data_loader = FaceLandmark77DataLoader(config)
else:
data_loader = SimpleMnistDataLoader(config)
else:
data_loader = SimpleMnistDataLoader(config)
print('Create the model.')
if hasattr(config, "model_name"):
if config.model_name == "mobile_net":
model = MobileNetV2Model(config)
else:
model = SimpleMnistModel(config)
else:
model = SimpleMnistModel(config)
print('Create the trainer')
trainer = SimpleMnistModelTrainer(model.model, data_loader, config)
print('Start training the model.')
trainer.train()
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# capture the config path from the run arguments
# then process the json configuration file
# try:
split = 10
training_size = []
best_model_train_loss = []
best_model_val_loss = []
best_model_train_acc = []
best_model_val_acc = []
best_model_train_precision = []
best_model_val_precision = []
best_model_train_recall = []
best_model_val_recall = []
best_model_train_f1 = []
best_model_val_f1 = []
best_model_learning_rate = []
best_model_nb_epoch = []
best_model_train_err = []
best_model_val_err = []
main_dir = ''
args = get_args()
for i in range(split):
config = process_config_VisTrainingSize(args.config, i)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
total_train_size = data_loader.get_train_size()
total_test_size = data_loader.get_test_size()
print("total_train_size: " + str(total_train_size))
print("total_test_size: " + str(total_test_size))
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
train_size = int(total_train_size / split) * (i + 1)
if i == split - 1:
print("train_size: " + str(total_train_size))
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
main_dir = config.main_dir
else:
print("train_size: " + str(train_size))
train_data = data_loader.get_train_data()
X_train = train_data[0][:train_size, :, :]
y_train = train_data[1][:train_size, :]
trainer = UtsClassificationTrainer(model.model, [X_train, y_train],
config)
print('Start training the model.')
trainer.train()
best_model_train_loss.append(trainer.best_model_train_loss)
best_model_val_loss.append(trainer.best_model_val_loss)
best_model_train_acc.append(trainer.best_model_train_acc)
best_model_val_acc.append(trainer.best_model_val_acc)
best_model_train_precision.append(trainer.best_model_train_precision)
best_model_val_precision.append(trainer.best_model_val_precision)
best_model_train_recall.append(trainer.best_model_train_recall)
best_model_val_recall.append(trainer.best_model_val_recall)
best_model_train_f1.append(trainer.best_model_train_f1)
best_model_val_f1.append(trainer.best_model_val_f1)
best_model_learning_rate.append(trainer.best_model_learning_rate)
best_model_nb_epoch.append(trainer.best_model_nb_epoch)
best_model_train_err.append(1 - trainer.best_model_train_acc)
best_model_val_err.append(1 - trainer.best_model_val_acc)
training_size.append(train_size)
print("ss")
metrics = {
"training_size": training_size,
"train_err": best_model_train_err,
"val_err": best_model_val_err
}
plot_trainingsize_metric(metrics,
main_dir + '/vis_overfit_trainingsize.png')
import numpy as np
import pandas as pd
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import tensorflow as tf
from utils.utils import log, get_args, prepare_directory, get_batch_count
from utils.preprocess import get_windowed_data
from models.EncDecEmbedding import Model
mpl.rcParams['agg.path.chunksize'] = 10000
# network parameters
# N_THREADS = 16
args = get_args()
# to be caculated
dataset = {
'train': np.empty((0, args.step_size)),
'validate': np.empty((0, args.step_size)),
'anomalous': np.empty((0, args.step_size)),
}
Y_LIMIT = [0, args.symbol_size + 1]
def read_dataset():
global dataset
for src, in zip(args.srcs):
def main():
# Capture the config path from the run arguments then process the json configuration file
try:
args = utils.get_args()
config = process_config(args)
except:
print("missing or invalid arguments")
args={}
args['config_file'] = 'configs/config_densenet.json'
args['mode'] = 'predict'
args['mode_ds'] = 'train_ds'
args['debug'] = 0
args = Bunch(args)
config = process_config(args)
if not os.path.exists(config.dataset_path_train):
print(config.dataset_path_train)
print('ERROR: Dataset not found')
exit(1)
# Initialize Logger
utils.logger_init(config, logging.DEBUG)
tf.logging.set_verbosity(tf.logging.DEBUG)
## Register signal handler
utils.signal_handler(config)
## Set seed values to reproduce results
random.seed(config.seed)
np.random.seed(config.seed)
tf.set_random_seed(config.seed)
## Create output dirs
# utils.remove_dirs([os.path.join(config.output_path, config.exp_name)])
utils.create_dirs([config.summary_dir, config.checkpoint_dir, config.tfrecords_path_train,
config.tfrecords_path_val, config.tfrecords_path_test])
## Save code
utils.save_code(config)
## Create tensorboard logger
# logger = TFLogger(sess, config)
# Config Keras
K.set_image_data_format('channels_last')
logging.debug('K.image_data_format {}'.format(K.image_data_format()))
## Configure GPU
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth=True
# gpu_train_fraction = 0.7
gpu_train_fraction = 1.0
if config.mode == 'train':
sess_config.gpu_options.per_process_gpu_memory_fraction = gpu_train_fraction
else:
sess_config.gpu_options.per_process_gpu_memory_fraction = 1 - gpu_train_fraction
tf.keras.backend.set_session(tf.Session(config=sess_config))
## TODO: _aSk Check why adding keras needed?
# config.checkpoint_dir = os.path.join(config.checkpoint_dir, 'keras')
timestamp_start = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
logging.debug('timestamp_start {}'.format(timestamp_start))
logging.debug('mode {}'.format(config.mode))
## Print updated configuration
logging.debug('\nConfiguration: \n{}'.format(config))
# sess=''
logger=''
## Create TF Records
if (config.mode == 'tfr'):
TFRecordsDensenet(config)
exit(0)
## Create data generator using TF Records
data = DataGeneratorDensenet(config)
## Get and set class weights in config
# utils.set_config_class_weights(config, data)
## Create model
model = ModelDensenet(config)
model.build_model(mode=config.mode, mode_ds=config.mode_ds)
with tf.Session(config=sess_config) as sess:
if config.debug == '1':
# tf.train.start_queue_runners(sess=sess)
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
# sess = tf_debug.LocalCLIDebugWrapperSession(sess, thread_name_filter="MainThread$")
## Initialze/Load variables
latest_checkpoint = tf.train.latest_checkpoint(config.checkpoint_dir)
if config.pre_trained_model:
model.saver.restore(sess, config.pre_trained_model)
logging.debug('Restored pre_trained_model_path {}'.format(config.pre_trained_model))
elif latest_checkpoint:
model.saver.restore(sess, latest_checkpoint)
logging.debug('Restored latest_checkpoint {}'.format(latest_checkpoint))
else:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
## Create Trainer
# trainer = TrainerDensenet(sess, model, data, config, logger)
trainer = TrainerDensenet_2(sess, model, data, config, logger)
## TRAINING
if (config.mode == 'train'):
trainer.train()
# trainer.train_and_eval()
# trainer.train_tf()
## EVALUATION
elif (config.mode == 'eval'):
last_checkpoint = None
while(True):
latest_checkpoint = tf.train.latest_checkpoint(config.checkpoint_dir)
if latest_checkpoint is None:
logging.debug('No checkpoint exists {}'.format(last_checkpoint))
time.sleep(5)
continue
if last_checkpoint == latest_checkpoint:
logging.debug('Sleeping latest_checkpoint {}'.format(latest_checkpoint))
time.sleep(5)
continue
last_checkpoint = latest_checkpoint
model.saver.restore(sess, latest_checkpoint)
if (config.mode_ds == 'train_ds'):
trainer.evaluate(mode_ds='train_ds')
elif (config.mode_ds == 'val_ds'):
trainer.evaluate(mode_ds='val_ds')
## PREDICTION
if (config.mode == 'predict'):
if(config.cascade == '0'):
# trainer.predict(mode_ds='train_ds')
# trainer.predict(mode_ds='val_ds')
trainer.predict(mode_ds='test_ds')
## PREDICTION CASCADE MODELS
elif(config.cascade == '1'):
trainer.predict_cascade(mode_ds='test_ds')
else:
logging.debug("ERROR: Unknown mode")
exit(1)
def main():
args = get_args()
model_name = 'WM_4layer'
Model = WM_4layer
Data = MNISTDataGenerator
Trainer = SFTrainer
config = process_config('/home/prashanth/synthfeedback/configs/wm.json',
model_name)
create_dirs([config.summary_dir, config.checkpoint_dir])
#Param search parameters
#attr = ['']
#var_vals = [1e-3, 1e-2, 1e-1, 1, 10]
#var_vals = [30,100]
N = 1
M = 3
#config.num_epoch=5
#print("\n\n",config.num_epoch)
T = config.num_epochs + 1
#config.gamma=250000
n_tags = 10
test_losses = np.zeros((N, M))
isnan = np.zeros((N, M))
metrics = np.zeros((N, M, T, n_tags))
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
for n in range(N):
#var_val = [var_vals[n]]
#set_hyperparameters(config, attr, var_val)
tf.reset_default_graph()
model = Model(config)
#print("\n\n",config.num_epoch)
data = Data(config)
#print('Hyperparameters: ' + attr[0] + ' = %f'%var_vals[n])
for m in range(M):
with tf.Session(config=tfconfig) as sess:
#options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
#run_metadata = tf.RunMetadata()
logger = LoggerNumpy(sess, config, model)
model.load(sess)
#trainer = Trainer(sess, model, data, config, logger, options, run_metadata)
trainer = Trainer(sess, model, data, config, logger)
try:
trainer.train()
except ValueError:
print("Method fails to converge for these parameters")
isnan[n, m] = 1
loss, acc = trainer.test()
metric = logger.get_data()
tags = logger.get_tags()
test_losses[n, m] = loss
metrics[n, m, :, :] = metric
#fetched_timeline = timeline.Timeline(run_metadata.step_stats)
#chrome_trace = fetched_timeline.generate_chrome_trace_format()
#with open('./timeline_02_n_%d_m_%d.json'%(n,m), 'w') as f:
# f.write(chrome_trace)
#Save after each run
#fn = os.path.join(config.summary_dir) + "WM_4layer(condnum).npz"
to_save = {
'test_losses': test_losses,
'metrics': metrics,
'isnan': isnan,
'tags': tags
}
#pickle.dump(to_save, open(fn, "wb"))
#np.savez(fn, test_losses=test_losses, metrics = metrics, isnan = isnan, tags = tags)
return metrics
def main():
args = get_args()
model_name = 'nodepert4_exact'
Model = NPModel4_ExactLsq_CorrectBatch
Data = MNISTDataGenerator
Trainer = SFTrainer
config = process_config('./configs/np.json', model_name)
create_dirs([config.summary_dir, config.checkpoint_dir])
#Param search parameters
attr = ['var_xi']
var_vals = [1e-4, 1e-3, 1e-2, 1e-1]
#var_vals = [1e-3]
N = len(var_vals)
M = 5
#M = 1
T = config.num_epochs + 1
n_tags = 10
test_losses = np.zeros((N, M))
isnan = np.zeros((N, M))
metrics = np.zeros((N, M, T, n_tags))
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
for n in range(N):
var_val = [var_vals[n]]
set_hyperparameters(config, attr, var_val)
tf.reset_default_graph()
model = Model(config)
data = Data(config)
print('Hyperparameters: ' + attr[0] + ' = %f' % var_vals[n])
for m in range(M):
with tf.Session(config=tfconfig) as sess:
logger = LoggerNumpy(sess, config, model)
model.load(sess)
trainer = Trainer(sess, model, data, config, logger)
try:
trainer.train()
except ValueError:
print("Method fails to converge for these parameters")
isnan[n, m] = 1
loss, acc = trainer.test()
metric = logger.get_data()
tags = logger.get_tags()
test_losses[n, m] = loss
metrics[n, m, :, :] = metric
#Save after each run
fn = os.path.join(
config.summary_dir
) + "2b_establish_convergence_feedforward_output_correctbatch.npz"
to_save = {
'test_losses': test_losses,
'metrics': metrics,
'isnan': isnan,
'tags': tags
}
pickle.dump(to_save, open(fn, "wb"))
return metrics
def main():
args = get_args()
model_name = 'nodepert4_fixedw_exact'
Model = NPModel4_ExactLsq_CorrectBatch
Data = MNISTDataGenerator
Trainer = SFTrainer
config = process_config('./configs/np.json', model_name)
create_dirs([config.summary_dir, config.checkpoint_dir])
#Param search parameters
attr = ['var_xi']
var_vals = [1e-3, 1e-2, 1e-1]
#var_vals = [1e-1]
N = len(var_vals)
M = 3
T = config.num_epochs+1
n_tags = 10
test_losses = np.zeros((N, M))
isnan = np.zeros((N, M))
metrics = np.zeros((N, M, T, n_tags))
tfconfig = tf.ConfigProto()
tfconfig.gpu_options.allow_growth = True
for n in range(N):
var_val = [var_vals[n]]
set_hyperparameters(config, attr, var_val)
tf.reset_default_graph()
model = Model(config)
data = Data(config)
print('Hyperparameters: ' + attr[0] + ' = %f'%var_vals[n])
for m in range(M):
with tf.Session(config=tfconfig) as sess:
#options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
#run_metadata = tf.RunMetadata()
logger = LoggerNumpy(sess, config, model)
model.load(sess)
#trainer = Trainer(sess, model, data, config, logger, options, run_metadata)
trainer = Trainer(sess, model, data, config, logger)
try:
trainer.train()
except ValueError:
print("Method fails to converge for these parameters")
isnan[n,m] = 1
loss, acc = trainer.test()
metric = logger.get_data()
tags = logger.get_tags()
test_losses[n,m] = loss
metrics[n,m,:,:] = metric
#See here: https://towardsdatascience.com/howto-profile-tensorflow-1a49fb18073d
#fetched_timeline = timeline.Timeline(run_metadata.step_stats)
#chrome_trace = fetched_timeline.generate_chrome_trace_format()
#with open('./timeline_02_n_%d_m_%d.json'%(n,m), 'w') as f:
# f.write(chrome_trace)
#Save after each run
fn = os.path.join(config.summary_dir) + "2_establish_convergence_feedforward_output_correctbatches.npz"
to_save = {
'test_losses': test_losses,
'metrics': metrics,
'isnan': isnan,
'tags': tags
}
pickle.dump(to_save, open(fn, "wb"))
return metrics
def main():
# capture the config path from the runtime arguments
# then process the json configuration file
args = get_args()
print("Reading config from {}".format(args.config))
config, _ = get_config_from_json(args.config)
# add summary and model directory
config = update_config_by_summary(config)
# if to remove the previous results, set -d 1 as a parameter
print('Whether to del the previous saved model', args.delete)
if args.delete == '1':
# delete existing checkpoints and summaries
print('Deleting existing models and logs from:')
print(config.summary_dir, config.checkpoint_dir)
remove_dir(config.summary_dir)
remove_dir(config.checkpoint_dir)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
"""Load data"""
# load global word, position and tag vocabularies
word_vocab = load_vocab(path=config.datadir + config.word_vocab_path,
mode='word')
position_vocab = load_vocab(path=config.datadir + config.pos_vocab_path,
mode='pos')
tag_vocab = load_vocab(path=config.datadir + config.tag_vocab_path,
mode='tag')
config.word_vocab_size = len(word_vocab)
config.pos_vocab_size = len(position_vocab)
config.tag_vocab_size = len(tag_vocab)
print('word vocab size:', config.word_vocab_size)
# create your data generator to load train data
x_path = config.datadir + config.train_path
train_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
train_loader.load_data()
# update the max length for each patient and each visit to be used in lstm
train_max_patient_len = train_loader.max_patient_len
train_max_visit_len = train_loader.max_visit_len
# create your data generator to load valid data
x_path = config.datadir + config.valid_path
valid_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
valid_loader.load_data()
valid_max_patient_len = valid_loader.max_patient_len
valid_max_visit_len = valid_loader.max_visit_len
# create your data generator to load test data
x_path = config.datadir + config.test_path
test_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
test_loader.load_data()
test_max_patient_len = test_loader.max_patient_len
test_max_visit_len = test_loader.max_visit_len
print("The max patient lengths for train, valid and test are {}, {}, {}".
format(train_max_patient_len, valid_max_patient_len,
test_max_patient_len))
print("The max visit lengths for train, valid and test are {}, {}, {}".
format(train_max_visit_len, valid_max_visit_len, test_max_visit_len))
# select the maximum lengths of visits and codes as the size of lstm
config.max_patient_len = max(
[train_max_patient_len, valid_max_patient_len, test_max_patient_len])
config.max_visit_len = max(
[train_max_visit_len, valid_max_visit_len, test_max_visit_len])
train_loader.pad_data(config.max_patient_len, config.max_visit_len)
valid_loader.pad_data(config.max_patient_len, config.max_visit_len)
test_loader.pad_data(config.max_patient_len, config.max_visit_len)
# add num_iter_per_epoch to config for trainer
config.train_size = train_loader.get_datasize()
config.valid_size = valid_loader.get_datasize()
config.test_size = test_loader.get_datasize()
config.num_iter_per_epoch = int(config.train_size / config.batch_size)
print("The sizes for train, valid and test are {}, {}, {}".format(
config.train_size, config.valid_size, config.test_size))
"""Run model"""
# create tensorflow session
# specify only using one GPU
tfconfig = tf.ConfigProto(device_count={'GPU': 1})
# allow the dynamic increase of GPU memory
tfconfig.gpu_options.allow_growth = True
# limit the maximum of GPU usage as 0.5
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.5
with tf.Session(config=tfconfig) as sess:
# create an instance of the model you want
model = Model(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = PredTrainer(sess, model, train_loader, valid_loader,
test_loader, config, logger)
# load model if exists
model.load(sess)
# here you train your model
trainer.train()
from model.hred import hred
from utils.Dialogue import Dialogue
import utils.utils as config
#temp data
enc_input_data = [[10, 5, 4, 7, 8], [8, 4, 6, 7, 0], [7, 10, 4, 0, 0],
[4, 8, 0, 0, 0]]
dec_input_data = [[1, 8, 6, 4, 5], [1, 10, 4, 8, 0], [1, 7, 5, 0, 0],
[1, 5, 0, 0, 0]]
target_data = [[8, 6, 4, 5, 2], [10, 4, 8, 2, 0], [7, 5, 2, 0, 0],
[5, 2, 0, 0, 0]]
enc_seq_len = [5, 4, 3, 2]
dec_seq_len = [5, 4, 3, 2]
max_len = 5
args = config.get_args(sys.argv[1:])
total_epoch = args.n_epochs
num_units = args.n_hidden
learning_rate = 0.001
voc_size = 11
hred_model = hred(voc_size, learning_rate, num_units)
fdtr = {
hred_model.enc_inputs: enc_input_data,
hred_model.dec_inputs: dec_input_data,
hred_model.targets: target_data,
hred_model.enc_seq_length: enc_seq_len,
hred_model.dec_seq_length: dec_seq_len,
hred_model.max_length: max_len
}
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir
])
print('Create the data generator.')
data_loader = MyDataLoader(config)
print('Create the model.')
model = MyModel(config)
print('Create the trainer')
trainer = MyModelTrainer(
model.model,
(data_loader.get_train_data(), data_loader.get_test_data()), config)
print('Start training the model.')
#trainer.train()
print('Plotting random samples.')
# training samples
fig1 = plt.figure(1)
fig1.clf()
rand_idx = np.random.choice(data_loader.X_train.shape[0], size=4)
y_pred = model.model.predict(data_loader.X_train[rand_idx, :, :, :])
for i, idx in enumerate(rand_idx):
img = data_loader.X_train[idx, :, :, :]
ax = fig1.add_subplot(2, 2, i + 1)
lm_x_true = data_loader.y_train[idx, 0::2]
lm_y_true = data_loader.y_train[idx, 1::2]
ax.imshow(np.transpose(img, axes=[1, 0, 2]).squeeze())
ax.plot(lm_x_true * config.data.IMAGE_SIZE,
lm_y_true * config.data.IMAGE_SIZE, 'gx')
lm_x_pred = y_pred[i, 0::2]
lm_y_pred = y_pred[i, 1::2]
ax.plot(lm_x_pred * config.data.IMAGE_SIZE,
lm_y_pred * config.data.IMAGE_SIZE, 'rx')
# print( np.mean((y_pred - y_train[rand_idx, :])**2) )
fig1.savefig('training_samples.png')
# test samples
fig2 = plt.figure(2)
fig2.clf()
rand_idx = np.random.choice(data_loader.X_test.shape[0], size=4)
y_pred = model.model.predict(data_loader.X_test[rand_idx, :, :, :])
for i, idx in enumerate(rand_idx):
img = data_loader.X_test[idx, :, :, :]
ax = fig2.add_subplot(2, 2, i + 1)
lm_x_true = data_loader.y_test[idx, 0::2]
lm_y_true = data_loader.y_test[idx, 1::2]
ax.imshow(np.transpose(img, axes=[1, 0, 2]).squeeze())
ax.plot(lm_x_true * config.data.IMAGE_SIZE,
lm_y_true * config.data.IMAGE_SIZE, 'gx')
lm_x_pred = y_pred[i, 0::2]
lm_y_pred = y_pred[i, 1::2]
ax.plot(lm_x_pred * config.data.IMAGE_SIZE,
lm_y_pred * config.data.IMAGE_SIZE, 'rx')
# print( np.mean((y_pred - y_train[rand_idx, :])**2) )
fig2.savefig('test_samples.png')
def main():
# capture the config path from the run arguments
# then process the json configuration file
# parse -c and put the wright config file ex. "-c configs/baseline.json"
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# LOADING DATA AND PREPROCESSING
path = './data/'
pimg = 'test_images/'
nfiles = len(os.listdir(path + pimg))
imgs = load_test_batch(path, pimg, nfiles, batch_size=config.test_size)
list_patches, SLIC_list = get_patches(imgs)
print("Got the patches.")
# flatten the data
patches_flat = []
[
patches_flat.append(patch) for patches_img in list_patches
for patch in patches_img
]
patches_flat = np.array(patches_flat)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# create an instance of the model you want
model = BaselineModel(config)
# TESTING
# load model if exists
model.load(sess)
feed_dict = {model.x: patches_flat, model.is_training: False}
pred = sess.run([model.pred], feed_dict=feed_dict)
labels_flat_copy = pred[0].copy()
for index in range(config.test_size):
slic_np = np.array(SLIC_list[index])
values = slic_np.flatten()
nl = len(set(values))
label = labels_flat_copy[:nl]
patches1 = []
i = 0
for l in label:
if l == 1:
patches1.append(i)
i += 1
labels_flat_copy = labels_flat_copy[nl:]
mm = imgs[index].copy()
for i in patches1:
mm[slic_np == i] = 255
fig, ax = plt.subplots(1,
2,
figsize=(10, 10),
sharex=True,
sharey=True)
ax[0].imshow(mark_boundaries(mm, SLIC_list[index]))
ax[1].imshow(mark_boundaries(imgs[index], SLIC_list[index]))
plt.show()
def setup_algorithms(server_args):
"""
Load datasets and pretrained models
"""
loaded_models = {}
datasets = None
abspath = os.path.abspath(".")
set_torch_seed(server_args.seed)
if "exp_path" in server_args and server_args["exp_path"] is not None:
abspath = os.path.abspath(server_args["exp_path"])
for builder_args in server_args.models:
original_args = copy.copy(builder_args)
assert 'continue_from' in builder_args, 'All "models" should have a "continue_from" entry.'
assert 'gpu' in builder_args, 'All "models" should have a specified "gpu" entry or "cpu" device.'
stdin_list = [
"--args_file",
os.path.join(abspath, builder_args["continue_from"], 'configs',
'config.json'), "--continue_from",
os.path.join(abspath, builder_args["continue_from"]), "--gpu",
builder_args['gpu'], "--seed", server_args.seed, "--dataset",
server_args.dataset, "--dataset_args",
json.dumps({
'dataset_version': server_args.version,
'data_path': server_args.data_path
})
]
builder_args, excluded_args, parser = get_args(stdin_list)
builder_args = bunch.bunchify(builder_args)
compressed_args = compress_args(bunch.unbunchify(builder_args), parser)
device = set_gpu(builder_args.gpu)
tasks = get_tasks(builder_args)
datasets = get_data(builder_args) if datasets is None else datasets
backbone = get_backbone(builder_args, device)
strategy = get_strategy(builder_args, device)
model = get_model(backbone, tasks, datasets, strategy, builder_args,
device)
compressed_args = compress_args(bunch.unbunchify(builder_args), parser)
print(" ----------------- FULL ARGS (COMPACT) ----------------")
pprint.pprint(compressed_args, indent=2)
print(" ------------------------------------------------------")
print(" ------------------ UNRECOGNISED ARGS -----------------")
pprint.pprint(excluded_args, indent=2)
print(" ------------------------------------------------------")
system = ExperimentBuilder(model, tasks, datasets, device,
builder_args)
system.load_pretrained()
model.set_mode('test')
if builder_args["model"] == 'simpleshot':
system.model.set_train_mean(system.datasets['train'])
name = original_args[
'name'] if 'name' in original_args else builder_args['model']
tie_breaker = 0
name_proposal = name
while name_proposal in loaded_models:
tie_breaker += 1
name_proposal = "{}({})".format(name, tie_breaker)
loaded_models[name_proposal] = system
return loaded_models, datasets
def main():
# Arguments
###########################################################################
try:
args = get_args()
config = process_config(args.config)
except:
logging.error("Missing or invalid arguments.")
exit(0)
# Logging
###########################################################################
logging.basicConfig(
filename=os.path.join("logs", config.exp_name + ".log"),
format="[%(asctime)s] - [%(levelname)s]: %(message)s",
filemode="a",
level=logging.DEBUG,
)
logging.info("Logging started.")
logging.info("Keras version: {}".format(keras_version))
# Session
###########################################################################
sess = tf.Session()
K.set_session(sess)
# create experiment related directories
###########################################################################
create_dirs([config.summary_dir, config.checkpoint_dir])
# Initialize the model
###########################################################################
model_formicID = load_model(config=config, num_species=97)
model_formicID = compile_model(model=model_formicID, config=config)
model_formicID = weights_load(
model=model_formicID,
weights=
"experiments/T97_CaAll_QuM_ShSti_AugM_D05_LR0001_E200_I4_def_clean/checkpoint/weights_55-1.76.hdf5",
)
# Training in batches with iterator
###########################################################################
history = trainer_dir(
model=model_formicID,
config=config,
callbacks=build_logger(config=config, model=model_formicID),
)
save_model(model=model_formicID,
filename="final_weights.hdf5",
config=config)
# Evaluation
###########################################################################
plot_history(history=history, config=config, theme="ggplot", save=None)
evaluator(model=model_formicID, config=config, test_dir=None)
# Testing
###########################################################################
Y_true, Y_pred, labels, species_dict = predictor(
model=model_formicID,
config=config,
# species_json="data/species_dict.json",
plot=True,
n_img=10,
n_cols=3,
)
predictor_reports(
Y_true=Y_true,
Y_pred=Y_pred,
config=config,
species_dict=species_dict,
target_names=labels,
digits=5,
)
plot_confusion_matrix(
Y_pred=Y_pred,
Y_true=Y_true,
config=config,
target_names=labels,
species_dict=species_dict,
title=None,
cmap="viridis",
normalize=True,
scores=True,
score_size=8,
save="confusion_matrix.png",
)
# Footer
###########################################################################
K.clear_session()
logging.info("Logging ended.")
def main():
args = get_args()
prepare_data()
word_vocab_config = {
"<UNK>": 0,
"<PAD>": 1,
"<start>": 2,
"<end>": 3,
"insert_start": "<SOS>",
"insert_end": "<EOS>",
"tokenization": "nltk",
"specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"],
"embedding_root": os.path.join(args.app_path, "data", "embedding",
"word"),
"embedding_type": "glove.840B",
"embedding_dim": 300
}
print("Reading Vocab", flush=True)
char_vocab_config = word_vocab_config.copy()
char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data",
"embedding", "char")
char_vocab_config["embedding_type"] = "glove_char.840B"
# TODO: build vocab out of dataset
# build vocab
itos, stoi, wv_vec = read_vocab(word_vocab_config)
itoc, ctoi, cv_vec = read_vocab(char_vocab_config)
char_embedding_config = {
"embedding_weights": cv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_char_embedding,
"bidirectional": args.bidirectional,
"cell_type": "gru",
"output_dim": 300
}
word_embedding_config = {
"embedding_weights": wv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_word_embedding
}
sentence_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": True,
"dropout": args.dropout,
}
pair_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
self_matching_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
pointer_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"dropout": args.dropout,
"residual": args.residual,
"rnn_cell": torch.nn.GRUCell
}
print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True)
train_cache = "./data/cache/SQuAD%s.pkl" % ("_debug" if args.debug else "")
dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug"
if args.debug else "")
train_json = args.train_json
dev_json = args.dev_json
train = read_dataset(train_json, itos, stoi, itoc, ctoi, train_cache,
args.debug)
dev = read_dataset(dev_json,
itos,
stoi,
itoc,
ctoi,
dev_cache,
args.debug,
split="dev")
dev_dataloader = dev.get_dataloader(args.batch_size_dev)
train_dataloader = train.get_dataloader(args.batch_size,
shuffle=True,
pin_memory=args.pin_memory)
trainer = Trainer(args, train_dataloader, dev_dataloader,
char_embedding_config, word_embedding_config,
sentence_encoding_config, pair_encoding_config,
self_matching_config, pointer_config)
trainer.train(args.epoch_num)
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
try:
RECEIVED_PARAMS = nni.get_next_parameter()
LOG.debug(RECEIVED_PARAMS)
PARAMS = generate_default_params()
PARAMS.update(RECEIVED_PARAMS)
args = get_args()
print(PARAMS['learning_rate'])
config = process_config_UtsClassification_bayes_optimization(
args.config, PARAMS['learning_rate'], PARAMS['model_name'],
PARAMS['batch_size'])
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
nni.report_final_result(evaluater.f1)
print('done')
except Exception as e:
LOG.exception(e)
raise
def main():
# create tensorflow session
sess = tf.Session()
try:
args = get_args()
config = process_config(args.config)
config.old_tfrecords = args.old_tfrecords
config.normalize_data = args.normalize_data
except Exception as e:
print("An error occurred during processing the configuration file")
print(e)
exit(0)
# create the experiments dirs
create_dirs(
[config.summary_dir, config.checkpoint_dir, config.config_file_dir])
# create your data generator
train_data = DataGenerator(config, sess, train=True)
test_data = DataGenerator(config, sess, train=False)
logger = Logger(sess, config)
print("using {} rollout steps".format(config.n_rollouts))
inp_rgb = tf.placeholder("float", [None, 120, 160, 7])
control = tf.placeholder("float", [None, 6])
gt_seg = tf.placeholder("float", [None, 120, 160])
pred = cnnmodel(inp_rgb, control)
predictions = tf.reshape(
pred, [-1, pred.get_shape()[1] * pred.get_shape()[2]])
labels = tf.reshape(gt_seg,
[-1, gt_seg.get_shape()[1] * gt_seg.get_shape()[2]])
global_step_tensor = tf.Variable(0, trainable=False, name='global_step')
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=predictions))
optimizer = tf.train.AdamOptimizer(
learning_rate=config.learning_rate).minimize(
loss, global_step=global_step_tensor)
with tf.variable_scope('cur_epoch'):
cur_epoch_tensor = tf.Variable(0, trainable=False, name='cur_epoch')
increment_cur_epoch_tensor = tf.assign(cur_epoch_tensor,
cur_epoch_tensor + 1)
with tf.variable_scope('global_step'):
cur_batch_tensor = tf.Variable(0, trainable=False, name='cur_batch')
increment_cur_batch_tensor = tf.assign(cur_batch_tensor,
cur_batch_tensor + 1)
next_element_train = train_data.get_next_batch()
next_element_test = test_data.get_next_batch()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
saver = tf.train.Saver(max_to_keep=config.max_checkpoints_to_keep)
latest_checkpoint = tf.train.latest_checkpoint(config.checkpoint_dir)
if latest_checkpoint:
print("Loading model checkpoint {} ...\n".format(latest_checkpoint))
saver.restore(sess, latest_checkpoint)
print("Model loaded")
def _process_rollouts(feature, train=True):
gt_merged_seg_rollout_batch = []
input_merged_images_rollout_batch = []
gripper_pos_vel_rollout_batch = []
for step in range(config.n_rollouts - 1):
if step < feature["unpadded_experiment_length"]:
obj_segments = feature["object_segments"][step]
""" transform (3,120,160,7) into (1,120,160,7) by merging the rgb,depth and seg masks """
input_merged_images = create_full_images_of_object_masks(
obj_segments)
obj_segments_gt = feature["object_segments"][step + 1]
gt_merged_seg = create_full_images_of_object_masks(
obj_segments_gt)[:, :, 3]
gripper_pos = feature["gripperpos"][step + 1]
gripper_vel = feature["grippervel"][step + 1]
gripper_pos_vel = np.concatenate([gripper_pos, gripper_vel])
gt_merged_seg_rollout_batch.append(gt_merged_seg)
input_merged_images_rollout_batch.append(input_merged_images)
gripper_pos_vel_rollout_batch.append(gripper_pos_vel)
if train:
retrn = sess.run(
[optimizer, loss, pred],
feed_dict={
inp_rgb: input_merged_images_rollout_batch,
control: gripper_pos_vel_rollout_batch,
gt_seg: gt_merged_seg_rollout_batch
})
return retrn[1], retrn[2]
else:
retrn = sess.run(
[loss, pred],
feed_dict={
inp_rgb: input_merged_images_rollout_batch,
control: gripper_pos_vel_rollout_batch,
gt_seg: gt_merged_seg_rollout_batch
})
""" sigmoid cross entropy runs logits through sigmoid but only during train time """
seg_data = sigmoid(retrn[1])
seg_data[seg_data >= 0.5] = 1.0
seg_data[seg_data < 0.5] = 0.0
return retrn[0], seg_data, gt_merged_seg_rollout_batch
for cur_epoch in range(cur_epoch_tensor.eval(sess), config.n_epochs + 1,
1):
while True:
try:
features = sess.run(next_element_train)
features = convert_dict_to_list_subdicts(
features, config.train_batch_size)
loss_batch = []
sess.run(increment_cur_batch_tensor)
for _ in range(config.train_batch_size):
for feature in features:
loss_train, _ = _process_rollouts(feature)
loss_batch.append([loss_train])
cur_batch_it = cur_batch_tensor.eval(sess)
loss_mean_batch = np.mean(loss_batch)
print('train loss batch {0:} is: {1:.4f}'.format(
cur_batch_it, loss_mean_batch))
summaries_dict = {config.exp_name + '_loss': loss_mean_batch}
logger.summarize(cur_batch_it,
summaries_dict=summaries_dict,
summarizer="train")
if cur_batch_it % config.test_interval == 1:
print("Executing test batch")
features_idx = 0 # always take first element for testing
features = sess.run(next_element_test)
features = convert_dict_to_list_subdicts(
features, config.test_batch_size)
loss_test_batch = []
for i in range(config.test_batch_size):
loss_test, seg_data, gt_seg_data = _process_rollouts(
features[features_idx], train=False)
loss_test_batch.append(loss_test)
loss_test_mean_batch = np.mean(loss_test_batch)
summaries_dict = {
config.exp_name + '_test_loss': loss_test_mean_batch
}
logger.summarize(cur_batch_it,
summaries_dict=summaries_dict,
summarizer="test")
print('test loss is: {0:.4f}'.format(loss_test_mean_batch))
if seg_data is not None and gt_seg_data is not None:
""" create gif here """
create_seg_gif(features,
features_idx,
config,
seg_data,
gt_seg_data,
dir_name="tests_during_training",
cur_batch_it=cur_batch_it)
if cur_batch_it % config.model_save_step_interval == 1:
print("Saving model...")
saver.save(sess, config.checkpoint_dir, global_step_tensor)
print("Model saved")
except tf.errors.OutOfRangeError:
break
sess.run(increment_cur_epoch_tensor)
return None
def main():
tf.set_random_seed(12345)
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
config.old_tfrecords = args.old_tfrecords
config.normalize_data = args.normalize_data
if args.n_epochs:
print("overwriting n_epochs in config file")
config.n_epochs = args.n_epochs
if args.mode is not None:
print("overwriting mode in config file")
config.mode = args.mode
if args.tfrecords_dir is not None:
print("overwriting tfrecord dir in config file")
config.mode = args.mode
if not hasattr(config, 'latent_state_noise'):
config.latent_state_noise = False
if not hasattr(config, 'do_multi_step_prediction'):
config.do_multi_step_prediction = False
if not hasattr(config, 'remove_edges'):
config.remove_edges = False
if not hasattr(config, 'batch_processing'):
config.batch_processing = False
if not hasattr(config, 'conv_layer_instance_norm'):
config.conv_layer_instance_norm = True
print(
"convolution layers are normalized with instance or batch norm"
)
if not hasattr(config, 'use_baseline_auto_predictor'):
config.use_baseline_auto_predictor = False
if not hasattr(config, 'nodes_get_full_rgb_depth'):
config.nodes_get_full_rgb_depth = False
if not hasattr(config, 'n_predictions'):
config.n_predictions = 1
if not hasattr(config, 'remove_pos_vel'):
config.remove_pos_vel = False
if not hasattr(config, 'edges_carry_segmentation_data'):
config.edges_carry_segmentation_data = False
if not hasattr(config, 'use_f_interact'):
config.use_f_interact = False
if not hasattr(config, "use_latent_motion_vectors"):
config.use_latent_motion_vectors = False
if not hasattr(config, "motion_vectors"):
config.motion_vectors = False
if config.normalize_data:
print("-- using normalized data as input --")
else:
print("-- using unnormalized data as input --")
model_class = locate("models.model_zoo." + config.model_zoo_file +
"." + config.model_zoo_file)
except Exception as e:
print("An error occurred during processing the configuration file")
print(e)
exit(0)
# create the experiments dirs
create_dirs(
[config.summary_dir, config.checkpoint_dir, config.config_file_dir])
# create tensorflow session
sess = tf.Session()
# create your data generator
train_data = DataGenerator(config, sess, train=True)
test_data = DataGenerator(config, sess, train=False)
model = model_class(config)
# create tensorboard logger
logger = Logger(sess, config)
if config.mode.startswith("train"):
only_test = False
else:
only_test = True
# create trainer and pass all the previous components to it
if "baseline_auto_predictor_extended_multistep_position" in config.model_zoo_file:
trainer = SingulationTrainerPredictorExtendedObjPosition(
sess,
model,
train_data,
test_data,
config,
logger,
only_test=False)
elif "predictor_extended" in config.model_zoo_file:
trainer = SingulationTrainerPredictorExtended(sess,
model,
train_data,
test_data,
config,
logger,
only_test=False)
elif "predictor_" in config.model_zoo_file:
trainer = SingulationTrainerPredictor(sess,
model,
train_data,
test_data,
config,
logger,
only_test=False)
elif "auto_encoder" in config.model_zoo_file:
trainer = SingulationTrainerAutoEncoder(sess,
model,
train_data,
test_data,
config,
logger,
only_test=False)
else:
trainer = SingulationTrainer(sess,
model,
train_data,
test_data,
config,
logger,
only_test=only_test)
print(" --- only initializing test graph --- ")
# load model if exists
model.load(trainer.sess)
if config.mode == "train_test":
print("--- Running TRAIN/TEST MODE ---")
trainer.train()
elif config.mode == "test":
print("--- Running TEST MODE ---")
trainer.test()
elif config.mode == "compute_metrics_over_test_set":
print("--- Running METRIC COMPUTATION OVER TEST SET ---")
trainer.compute_metrics_over_test_set()
elif config.mode == "compute_metrics_over_test_set_multistep":
print("--- Running METRIC COMPUTATION OVER TEST SET (MULTISTEP) ---")
trainer.compute_metrics_over_test_set_multistep()
elif config.mode == "test_specific_exp_ids":
print("--- Running SPECIFIC EXP ID'S TEST MODE ---")
trainer.test_specific_exp_ids()
elif config.mode == "store_latent_vectors":
print("--- Running store latent vectors mode ---")
trainer.store_latent_vectors()
elif config.mode == "save_encoder_vectors":
print("--- Running SAVE ENCODER VECTORS ---")
trainer.save_encoder_vectors(train=True)
def main():
# 获取配置文件路径
# 运行：python main.py -c configs/fusion_config.json #for MCF-3D-CNN
# Or: python main.py -c configs/3dcnn_config.json #for 3DCNN
# 可视化: tensorboard --logdir=experiments/2018-04-23/MCF-3D CNN/logs
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
create_dirs([config.tensorboard_log_dir, config.checkpoint_dir, 'experiments', 'experiments/img',\
'experiments/img/all', 'experiments/models'])
for fusion_type in P:
for Fusion in S:
No = -1
max_score = 0
max_sens = 0
max_prec = 0
max_f1 = 0
save_tag = str(Fusion) + fusion_type + '_012'
#重复10次实验
for i in range(config.repeat):
# 载入数据
print('Create the data generator.')
data_loader = LiverDataLoader(config)
# 建立模型
print('Create the model.')
model = LiverModel(config, fusion_type, Fusion)
# 训练模型、评估模型
print('Create the trainer')
trainer = LiverModelTrainer(model.model, data_loader.get_data(Fusion), config)
print('Start training the model.')
trainer.train(fusion_type, Fusion, i, max_score)
score, sens, prec, f1 = trainer.getResults('avg')
# Record the results
Acc.append(score)
Sens.append(sens)
Prec.append(prec)
F1.append(f1)
TestTime.append(trainer.testTime)
mul_acc, mul_sens, mul_spec, mul_auc = trainer.getResults('mul')
#OneVsAll
Mul_acc.append(mul_acc)
Mul_sens.append(mul_sens)
Mul_spec.append(mul_spec)
Mul_auc.append(mul_auc)
# Record best result
if score > max_score:
No = i
max_score = score
max_sens = sens
max_prec = prec
max_f1 = f1
print(max_score, max_sens, max_prec, max_f1)
print('Test time:' + str(TestTime))
# Save Overall
fp = open('experiments/results.txt', 'ab+')
fp.write(save_tag + '\nAvg @ Acc:%.4f+-%.4f Sens:%.4f+-%.4f Prec:%.4f+-%.4f F1:%.4f+-%.4f\n'\
%(np.mean(Acc), np.std(Acc), np.mean(Sens), np.std(Sens),
np.mean(Prec), np.std(Prec), np.mean(F1), np.std(F1)))
fp.write('Test time: %.4f +- %.4f\n'%(np.mean(TestTime), np.std(TestTime)))
fp.write('Best@No%d Acc:%.4f Sens:%.4f Prec:%.4f F1:%.4f\n\n'%(No, max_score, max_sens, max_prec, max_f1))
fp.close()
#Save OneVsAll
f = open('experiments/img/oneVsAll.txt', 'ab+')
Acc_means, Acc_stds = np.mean(Mul_acc, 0), np.std(Mul_acc, 0)
f.write('\nAcc: %.4f+-%.4f\t'%(Acc_means[0], Acc_stds[0]))
f.write('%.4f+-%.4f\t'%(Acc_means[1], Acc_stds[1]))
f.write('%.4f+-%.4f\t'%(Acc_means[2], Acc_stds[2]))
Sens_means, Sens_stds = np.mean(Mul_sens, 0), np.std(Mul_sens, 0)
f.write('\nSens: %.4f+-%.4f\t'%(Sens_means[0], Sens_stds[0]))
f.write('%.4f+-%.4f\t'%(Sens_means[1], Sens_stds[1]))
f.write('%.4f+-%.4f\t'%(Sens_means[2], Sens_stds[2]))
Spec_means, Spec_stds = np.mean(Mul_spec, 0), np.std(Mul_spec, 0)
f.write('\nSpec: %.4f+-%.4f\t'%(Spec_means[0], Spec_stds[0]))
f.write('%.4f+-%.4f\t'%(Spec_means[1], Spec_stds[1]))
f.write('%.4f+-%.4f\t'%(Spec_means[2], Spec_stds[2]))
AUC_means, AUC_stds = np.mean(Mul_auc, 0), np.std(Mul_auc, 0)
f.write('\nAUC: %.4f+-%.4f\t'%(AUC_means[0], AUC_stds[0]))
f.write('%.4f+-%.4f\t'%(AUC_means[1], AUC_stds[1]))
f.write('%.4f+-%.4f\t'%(AUC_means[2], AUC_stds[2]))
f.close()
# 绘制平均ROC曲线
plot_avg_roc('experiments/img/all/'+save_tag, 2, 3, 'Class0')
plot_avg_roc('experiments/img/all/'+save_tag, 6, 7, 'Class1')
plot_avg_roc('experiments/img/all/'+save_tag, 10, 11, 'Class2')
def main():
args = get_args()
prepare_data()
word_vocab_config = {
"<UNK>": 0,
"<PAD>": 1,
"<start>": 2,
"<end>": 3,
"insert_start": "<SOS>",
"insert_end": "<EOS>",
"tokenization": "nltk",
"specials": ["<UNK>", "<PAD>", "<SOS>", "<EOS>"],
"embedding_root": os.path.join(args.app_path, "data", "embedding",
"word"),
"embedding_type": "glove.840B",
"embedding_dim": 300
}
print("Reading Vocab", flush=True)
char_vocab_config = word_vocab_config.copy()
char_vocab_config["embedding_root"] = os.path.join(args.app_path, "data",
"embedding", "char")
char_vocab_config["embedding_type"] = "glove_char.840B"
# TODO: build vocab out of dataset
# build vocab
itos, stoi, wv_vec = read_vocab(word_vocab_config)
itoc, ctoi, cv_vec = read_vocab(char_vocab_config)
char_embedding_config = {
"embedding_weights": cv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_char_embedding,
"bidirectional": args.bidirectional,
"cell_type": "gru",
"output_dim": 300
}
word_embedding_config = {
"embedding_weights": wv_vec,
"padding_idx": word_vocab_config["<UNK>"],
"update": args.update_word_embedding
}
sentence_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": True,
"dropout": args.dropout,
}
pair_encoding_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
self_matching_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"bidirectional": args.bidirectional,
"dropout": args.dropout,
"gated": True,
"mode": "GRU",
"rnn_cell": torch.nn.GRUCell,
"attn_size": args.attention_size,
"residual": args.residual
}
pointer_config = {
"hidden_size": args.hidden_size,
"num_layers": args.num_layers,
"dropout": args.dropout,
"residual": args.residual,
"rnn_cell": torch.nn.GRUCell
}
print("DEBUG Mode is ", "On" if args.debug else "Off", flush=True)
dev_cache = "./data/cache/SQuAD_dev%s.pkl" % ("_debug"
if args.debug else "")
test_json = args.test_json
test = read_dataset(test_json,
itos,
stoi,
itoc,
ctoi,
dev_cache,
args.debug,
split="dev")
test_dataloader = test.get_dataloader(args.batch_size_dev)
tester = Tester(args, test_dataloader, char_embedding_config,
word_embedding_config, sentence_encoding_config,
pair_encoding_config, self_matching_config, pointer_config)
result = tester.test()
json.dump(result, open('prediction.json', 'w'))
pd.DataFrame([[id, ' '.join([str(j) for j in range(ans[0], ans[1])])]
for id, ans in result.items()],
columns=['id', 'answer']).to_csv('prediction.csv',
index=False)
def main():
args = get_args()
#model_name = 'nodepert_ae5_sgd'
#model_name = 'nodepert_ae5_bpauto'
model_name = 'nodepert_ae5_adam'
#model_name = 'nodepert_ae5_bpself'
#model_name = 'nodepert_ae5_faauto'
#model_name = 'nodepert_ae5_faself'
#Model = AENPModel5
Model = AENPModel5_ADAM
#Model = AENPModel5_ExactLsq
#Model = AENPModel5_ExactLsq_BPAuto
#Model = AENPModel5_ExactLsq_BPSelf
#Model = AENPModel5_ExactLsq_FAAuto
#Model = AENPModel5_ExactLsq_FASelf
Data = MNISTDataGenerator
Trainer = AESFTrainer
config = process_config('./configs/np.json', model_name)
create_dirs([config.summary_dir, config.checkpoint_dir])
#Param search parameters
attr = ['var_xi', 'learning_rate', 'lmda_learning_rate']
attr_ranges = [[-4, -1], [-6, -3], [-6, -3]]
log_scale = [True, True, True]
N = 20
#M = 5
M = 1
T = config.num_epochs + 1
n_tags = 13
test_losses = np.zeros((N, M))
isnan = np.zeros((N, M))
metrics = np.zeros((N, M, T, n_tags))
params = []
for n in range(N):
param = set_random_hyperparameters(config, attr, attr_ranges,
log_scale)
params.append(param)
tf.reset_default_graph()
model = Model(config)
data = Data(config)
print('Hyperparameters: ' + ' '.join(
[attr[ii] + ' = %f' % param[ii] for ii in range(len(attr))]))
for m in range(M):
with tf.Session() as sess:
logger = LoggerNumpy(sess, config, model)
model.load(sess)
trainer = Trainer(sess, model, data, config, logger)
try:
trainer.train()
except ValueError:
print("Method fails to converge for these parameters")
isnan[n, m] = 1
loss, acc = trainer.test()
metric = logger.get_data()
tags = logger.get_tags()
test_losses[n, m] = loss
metrics[n, m, :, :] = metric
#Save after each run
fn = os.path.join(
config.summary_dir) + "3_autoencoder_correctbatch_hyperparam.npz"
to_save = {
'attr': attr,
'params': params,
'test_losses': test_losses,
'metrics': metrics,
'isnan': isnan,
'tags': tags
}
pickle.dump(to_save, open(fn, "wb"))
return metrics
def main():
# capture the config path from the run arguments
# then process the json configuration file
# try:
# args = get_args()
# phase = args.phase
# config = process_config(args.config)
# except:
# print("missing or invalid arguments")
# exit(0)
args = get_args()
phase = args.phase
config = process_config(args.config)
# create the experiments dirs
create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])
print('Create the data generator.')
data_loader = PerfectStoreDataLoader(config)
# import time
# t = []
# i = 0
# start = time.time()
# for images, label in data_loader.get_train_generator():
# t.append(time.time() - start)
# print(t[-1])
# i = i + 1
# if i > 10:
# break
# start = time.time()
# print('total: {}'.format(sum(t)))
# print('avg: {}'.format(sum(t)/len(t)))
#
# return
if phase == 'train':
print('Create the model.')
model = TripletLossModel(config)
print('Create the trainer')
trainer = TripletLossModelTrainer(model.model, data_loader, config)
print('Start training the model.')
trainer.train()
else:
import numpy as np
from scipy.spatial.distance import cdist
from sklearn import metrics
print('Load the model.')
model = TripletLossModel(config)
model.load(config.weight_file)
########################################
ref_images, ref_labels = data_loader.get_reference_data()
ref_embeddings = model.predict(ref_images,
batch_size=config.batch_size,
verbose=1)
eval_embeddings = []
eval_labels = []
for (images, labels) in data_loader.get_val_generator():
embeddings = model.predict(images)
eval_embeddings.append(embeddings)
eval_labels.append(labels)
eval_embeddings = np.concatenate(eval_embeddings)
eval_labels = np.concatenate(eval_labels)
eval_categories = np.unique(eval_labels)
pairwise_distance = cdist(eval_embeddings,
ref_embeddings,
metric=config.distance_metric)
predictions = ref_labels[np.argmin(pairwise_distance, axis=1)]
result = {
'val_accuracy':
metrics.accuracy_score(eval_labels, predictions),
'val_precision':
metrics.precision_score(eval_labels,
predictions,
labels=eval_categories,
average='macro'),
'val_recall':
metrics.recall_score(eval_labels,
predictions,
labels=eval_categories,
average='macro'),
'val_f1_score':
metrics.f1_score(eval_labels,
predictions,
labels=eval_categories,
average='macro')
}
print(' Result: {}'.format(' - '.join(
['{}: {}'.format(key, value) for key, value in result.items()])))
def main():
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
# except:
# print("missing or invalid arguments")
# exit(0)
if config.gpu_mode is True and not torch.cuda.is_available(
): #虽然开启gpu模式，但是找不到GPU
raise Exception("No GPU found, please run without --gpu_mode=False")
# create an instance of the model you want
model = Net(config)
# set the logger
log_dir = os.path.join(config.save_dir, 'logs_' + config.exp_name)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# logger = Logger(log_dir)
logger = None
# create your data generator
# data_train = DataGenerator(config, 'debug').load_dataset()
# create your data generator
data_test = DataGenerator(config, 'debug').load_dataset()
data_train = None
# create trainer and pass all the previous components to it
# trainer = Trainer(model, config, data_train, logger, data_test)
# trainer.train_test()
# # create tester and pass all the previous components to it
# 使用最后一个模型：在trainer.py中使用load_model函数
# 使用非最后一个模型：在base_model模块中指定特定模型，并在trainer.py中使用load_spec_model函数
tester = Tester(model, config, data_train, logger, data_test)
with torch.no_grad():
tester.test()
from utils.utils import set_torch_seed, set_gpu, get_tasks, get_data, get_model, get_backbone, get_strategy
from utils.utils import compress_args, get_args, torch_summarize
from utils.builder import ExperimentBuilder
from utils.bunch import bunch
import sys
import pprint
if __name__ == '__main__':
args, excluded_args, parser = get_args()
args = bunch.bunchify(args)
set_torch_seed(args.seed)
device = set_gpu(args.gpu)
datasets = get_data(args)
tasks = get_tasks(args)
backbone = get_backbone(args, device)
strategy = get_strategy(args, device)
model = get_model(backbone, tasks, datasets, strategy, args, device)
compressed_args = compress_args(bunch.unbunchify(args), parser)
print(" ----------------- FULL ARGS (COMPACT) ----------------")
pprint.pprint(compressed_args, indent=2)
print(" ------------------------------------------------------")
print(" ------------------ UNRECOGNISED ARGS -----------------")
pprint.pprint(excluded_args, indent=2)
print(" ------------------------------------------------------")
system = ExperimentBuilder(model, tasks, datasets, device, args)
system.load_pretrained()
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
# capture the config path from the run arguments
# then process the json configuration file
# try:
split = 10
training_size = []
accuracy = []
precision = []
recall = []
f1 = []
main_dir = ''
args = get_args()
for i in range(split):
config = process_config_VisOverfit(args.config, i)
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir, config.log_dir, config.result_dir
])
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
total_train_size = data_loader.get_train_size()
total_test_size = data_loader.get_test_size()
print("total_train_size: " + str(total_train_size))
print("total_test_size: " + str(total_test_size))
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
train_size = int(total_train_size / split) * (i + 1)
if i == split - 1:
print("train_size: " + str(total_train_size))
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
main_dir = config.main_dir
else:
print("train_size: " + str(train_size))
train_data = data_loader.get_train_data()
X_train = train_data[0][:train_size, :, :]
y_train = train_data[1][:train_size, :]
trainer = UtsClassificationTrainer(model.model, [X_train, y_train],
config)
print('Start training the model.')
trainer.train()
print('Create the evaluater.')
evaluater = UtsClassificationEvaluater(trainer.best_model,
data_loader.get_test_data(),
data_loader.get_nbclasses(),
config)
print('Start evaluating the model.')
evaluater.evluate()
training_size.append(train_size)
accuracy.append(evaluater.acc)
precision.append(evaluater.precision)
recall.append(evaluater.recall)
f1.append(evaluater.f1)
print("ss")
metrics = {
"accuracy": accuracy,
"precision": precision,
"recall": recall,
"f1": f1,
"training_size": training_size
}
plot_trainingsize_metric(metrics,
main_dir + 'vis_overfit_trainingsize.png')
def predict():
try:
args = get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
# Pre-process images
print('Pre-process images:')
process_ims(input_dir='../samples/*',
output_dir='../samples/processed/',
out_height=config.im_height)
# Get Model
model_types = import_module('models.' + config.architecture + '_model')
Model = getattr(model_types, 'Model')
# create tensorflow session
sess = tf.Session()
# create your data generator
data_loader = DataGenerator(config,
eval_phase=True,
eval_on_test_data=False)
# create instance of the model you want
model = Model(data_loader, config)
# load the model from the best checkpoint
model.load(sess, config.best_model_dir)
x, length, lab_length, y, is_training = tf.get_collection('inputs')
pred = model.prediction
# initialize dataset
data_loader.initialize(sess, is_train=False)
# Progress bar
tt = range(data_loader.num_iterations_val)
# Iterate over batches
predictions, filenames = [], sorted(glob.glob('../samples/*'))
for _ in tqdm(tt):
preds_sparse = sess.run([pred], feed_dict={is_training: False})
# Map numeric predictions with corresponding character labels
preds_out = np.zeros(preds_sparse[0][0][0].dense_shape)
for idx, val in enumerate(preds_sparse[0][0][0].indices):
preds_out[val[0]][val[1]] = preds_sparse[0][0][0].values[idx]
predictions += [
''.join([data_loader.char_map_inv[j] for j in preds_out[i]])
for i in range(preds_out.shape[0])
]
print('\nPredictions:')
[
print('{}: {}'.format(name[name.rfind('/') + 1:], model_pred))
for name, model_pred in zip(filenames, predictions)
]
shutil.rmtree('../samples/processed/')
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
config = process_config(args.config)
except Exception as e:
print("missing or invalid arguments %s" % e)
exit(0)
# create your data generator
#data = SemDataGenerator(config)
#data = MSRPGenerator(config)
data = ATECGenerator(config)
if args.step == "build_data":
print("build data....")
data.build_data()
elif args.step == "train":
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# load word2vec
config.embedding = data.get_trimmed_glove_vectors()
model = SelfAttESIM(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = SentSemTrainer(sess, model, data, config, logger)
#load model if exists
#model.load(sess)
# here you train your model
trainer.train()
elif args.step == "tune":
import itertools
tune_num = 0
param_names = config["parameter_tune"].keys()
print(param_names)
cand_params = [
config["parameter_tune"][pname] for pname in param_names
]
print(cand_params)
for params in itertools.product(*cand_params):
print(params)
for i, param_name in enumerate(param_names):
config[param_name] = params[i]
#print(config)
data = ATECGenerator(config)
create_dirs([config.summary_dir, config.checkpoint_dir])
sess = tf.Session()
config.embedding = data.get_trimmed_glove_vectors()
model = ESIM(config)
logger = Logger(sess, config)
trainer = SentSemTrainer(sess, model, data, config, logger)
trainer.train()
tf.reset_default_graph()
else:
print("no support step!!")
def main():
# capture the config path from the run arguments
# then process the json configuration file
global image
try:
args = get_args()
config = process_config(args.config)
image = process_image(args.image) if args.image is not 'None' else None
except:
print("missing or invalid arguments (check correct config or image paths)")
exit(0)
# create the experiments dirs
create_dirs([config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir, config.callbacks.history_dir])
print('Create the data generator.')
data_loader = ConvEMnistDataLoader(config) # Load data set
print('Some data visualization')
X_train, y_train = data_loader.get_train_data() # Get training data
print("ytrain")
print(y_train.shape)
mapp = data_loader.get_map() # Get map dictionary (Refer to emnist-balanced-mapping.txt file)
data_visualizer = SimpleMnistDataVisualizer(X_train, y_train, mapp)
data_visualizer.plot_first_digit() # Plot first character of training set
data_visualizer.plot_range() # Plot several characters
print('Create the model.')
model = XyEMnistModelX(config) # Create the model based on configuration file
print("Model Summary")
model.model.summary() # Print a summary of the model with the respective parameters
# Custom weight to use instead of training the model
weight = config.evaluator.weight
print('Create the trainer')
trainer = ConvMnistModelTrainer(model.model, data_loader.get_train_data(), config)
if not config.evaluator.custom_weight:
print('Start training the model.')
# if not config.evaluator.custom_weight:
trainer.train()
print("Plot loss and accuracy in training model")
data_visualizer.plot_loss_acc()
print("Finish training")
print("Predict")
predictor = ConvMnistDataPredictor(model.model, data_loader.get_test_data(), mapp, config, weight)
predict_image = image if image is not None else './test_images/data_representation/0.png'
predicted_values = predictor.ocr(predict_image)
print("Predicted values")
print(predicted_values)
# predictor.predict3('./test_images/h/1.png')
# predictor.predict_from_data_set()
"""
Evaluate model with test set
"""
predictor.evaluate_model()
predictor.confusion_matrix()
def main():
""" Runs the main descriptive stadistict about stocks and also get optimal portafolio
"""
t0 = dt.datetime.now()
args = utils.get_args()
all_config = yaml.safe_load(open(args.config_file_path, "r"))
DATA = all_config['output']['data_folder']
input_data_path = all_config['stocks']['data']
logger = logger_utils.log_creator(all_config['output']['log_folder'], log_name='get_stats')
start_date, end_date = utils.get_start_end(all_config)
df_ticks_path = os.path.join('./src/data', input_data_path)
logger.info("Reading tick and weights from %s" % df_ticks_path)
weights = utils.get_ticks_data(df_ticks_path)
data_path = utils.filename_maker('stocks_', DATA, start_date, end_date)
data_sp_path = utils.filename_maker('sp500_', DATA, start_date, end_date)
df = pd.read_csv(data_path)
sp = pd.read_csv(data_sp_path)
df['Date'] = pd.to_datetime(df['Date'])
df.set_index('Date', inplace=True)
df_pc = df.pct_change().dropna()
sp['Date'] = pd.to_datetime(sp['Date'])
sp.set_index('Date', inplace=True)
sp_pc = sp.pct_change().dropna()
weights_np = weights['WEIGHT'].to_numpy()
anual_cov_matrix = df_pc.cov()*252
volatilidad_por_anual = np.sqrt(np.dot(weights_np.T, np.dot(anual_cov_matrix, weights_np)))
logger.info("Anual portafolio volatility is %.2f" % volatilidad_por_anual)
portafolio_anual_return = np.sum(df_pc.mean()*weights_np)*252
logger.info("Anual portafolio return is %.2f" % portafolio_anual_return)
logger.info("Mean historical return for each stock %s" % round((df_pc.mean()*252),2))
logger.info("Anual volatility for each stock %s" % round(np.std(df_pc)*np.sqrt(252),2))
# np.sum(df_pc.mean()*weights['WEIGHT'].to_numpy())*252
ticks = weights['TICK'].to_list()
skew_list = []
kurtosis_list = []
shapiro_list = []
annual_vol = []
annual_returns = []
for tk in ticks:
skewness = np.round(df_pc[tk].skew(), 3)
kurt = np.round(df_pc[tk].kurtosis() + 3, 3)
shap = np.round(shapiro(df_pc[tk])[1], 3)
vol = np.round(df_pc[tk].std()*np.sqrt(252), 3)
rtn = np.round((df_pc[tk].mean()*252), 3)
skew_list.append(skewness)
kurtosis_list.append(kurt)
shapiro_list.append(shap)
annual_vol.append(vol)
annual_returns.append(rtn)
logger.info("This is the summary of the stocks regarding the anual return, anual volatility, kurtosis, shapiro and skew.")
stocks_summary = pd.DataFrame({'STOCK': ticks,
'SKEW': skew_list,
'KURTOSIS': kurtosis_list,
'SHAPIRO': shapiro_list,
'ANNUAL_VOL': annual_vol,
'ANNUAL_RETURN': annual_returns})
stocks_summary.set_index('STOCK', inplace=True)
logger.info(stocks_summary)
logger.info("Lets now calculate the anual covariance between stoks")
cov_matriz = df_pc.cov()*252
logger.info(cov_matriz)
logger.info("Using Python Portafolio")
mu = expected_returns.mean_historical_return(df)
sigma = risk_models.sample_cov(df)
ef = EfficientFrontier(mu, sigma)
logger.info("Showing portafolio with max sharpe rate")
raw_weights_maxsharpe = ef.max_sharpe()
cleaned_weights_maxsharpe = ef.clean_weights()
logger.info(cleaned_weights_maxsharpe)
# Show portfolio performance
logger.info(ef.portfolio_performance(verbose=True))
desire_return = 0.20
ef.efficient_return(desire_return)
logger.info("Calculating portafolio which should bring a return of %s" % desire_return)
logger.info(ef.clean_weights())
logger.info("Showing portafolio with lowest risk for a return of %s" % desire_return)
raw_weights_minvol = ef.min_volatility()
cleaned_weights_minvol = ef.clean_weights()
logger.info(cleaned_weights_minvol)
logger.info(ef.portfolio_performance(verbose=True))
t1 = dt.datetime.now()
# Read image paths
image_paths = utils_image.get_images_path_list_from_dir(
os.path.join(dataset_path, label_name), img_format='jpg')
images_count = len(image_paths)
image_paths_list = image_paths_list + image_paths
# Create labels
gt_labels = gt_labels + [label_no] * images_count
return image_paths_list, gt_labels
def wrap_data(self, key, value, output_path):
if key == 'image':
# Pass output_path as param in order to save images
return self.wrap_data_image(value, output_path)
elif key == 'label':
return tf.train.Feature(int64_list=tf.train.Int64List(
value=[value]))
else:
print('ERROR: Unknown type of data: {}'.format(type(value[0])))
exit(1)
### MAIN ###
if __name__ == '__main__':
args = utils.get_args()
config = process_config(args.config)
tfrecords_ks = TFRecordsKnifeySpoony(config)
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '2'
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
bayes_config = {
"algorithm": "bayes",
"parameters": {
# "model": {"type": "categorical", "values": ['cnn','mlp']},
"learning_rate": {
"type": "float",
"min": 0.001,
"max": 0.01
},
# "batch_size": {"type": "integer", "min": 16, "max": 32},
# "num_epochs": {"type": "integer", "min": 5, "max": 10},
},
"spec": {
"maxCombo": 10,
"objective": "minimize",
"metric": "test_f1",
"minSampleSize": 100,
"retryAssignLimit": 0,
},
"trials": 1,
"name": "Bayes",
}
opt = Optimizer(bayes_config,
api_key=config.comet_api_key,
project_name=config.exp_name)
for exp in opt.get_experiments():
args = get_args()
# config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('model'),exp.get_parameter('learning_rate'),
# exp.get_parameter('batch_size'), exp.get_parameter('num_epochs'))
config = process_config_UtsClassification_bayes_optimization(
args.config, exp.get_parameter('learning_rate'))
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
# print('Create the evaluater.')
# evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
# config)
#
# print('Start evaluating the model.')
# evaluater.evluate()
exp.log_metric("test_f1", trainer.best_model_val_loss)
print('done')
