def main():
config, model, dataset_train, _, dataset_test = argparser(is_train=False)
evaler = Evaler(config, model, dataset_test)
log.warning("dataset: %s", config.dataset)
evaler.eval_run()
def main():
config, model, dataset_train, dataset_test = argparser(is_train=False)
trainer = Trainer(config, model, dataset_train, dataset_test)
log.warning("dataset: %s", config.dataset)
trainer.train()
def main():
config, model, dataset_train, dataset_train_unlabel, dataset_test = argparser(is_train=True)
trainer = Trainer(config, model, dataset_train, dataset_train_unlabel, dataset_test)
log.warning("dataset: %s, learning_rate_g: %f, learning_rate_d: %f",
config.dataset, config.learning_rate_g, config.learning_rate_d)
trainer.train()
def main():
print("step0")
config, model, dataset_train, dataset_test = argparser(is_train=True)
print("step1")
trainer = Trainer(config, model, dataset_train, dataset_test)
log.warning("dataset: %s, learning_rate_g: %f, learning_rate_d: %f",
config.dataset, config.learning_rate_g, config.learning_rate_d)
trainer.train()
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
config, model, dataset_train, dataset_test = argparser(is_train=False)
evaler = Evaler(config, model, dataset_test)
log.warning("dataset: %s", config.dataset)
evaler.eval_run()
def main(_):
def shutdown(signal, frame):
logger.warning('Received signal %s: exiting', signal)
sys.exit(128 + signal)
signal.signal(signal.SIGHUP, shutdown)
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
args = config.argparser()
run(args)
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
config, model, dataset_train, dataset_test = argparser(is_train=True)
trainer = Trainer(config, model, dataset_train, dataset_test)
log.warning("dataset: %s, learning_rate_g: %f, learning_rate_d: %f",
config.dataset, config.learning_rate_g, config.learning_rate_d)
trainer.train()
def main():
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
config, model, dataset_train, dataset_val, dataset_test = argparser(
is_train=True)
trainer = Trainer(config, model, dataset_train)
log.warning("dataset_path: %s, learning_rate: %f", config.dataset_path,
config.learning_rate)
trainer.train()
def main():
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
config, model, dataset_train, dataset_val, dataset_test = argparser(is_train=False)
log.warning("dataset path: %s", config.dataset_path)
evaler_val = Evaler(config, model, dataset_val, 'val')
evaler_val.eval_run()
evaler_train = Evaler(config, model, dataset_train, 'train')
evaler_train.eval_run()
evaler_train = Evaler(config, model, dataset_test, 'test')
evaler_train.eval_run()
def main():
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
config, model, dataset_train, dataset_val, dataset_test = argparser(
is_train=False)
log.warning("dataset path: %s", config.dataset_path)
#viewer_val = Visualizer(config, model, dataset_val, 'val')
#viewer_val.vis_run()
#config.batch_size = viewer_val.batch_size
#viewer_train = Visualizer(config, model, dataset_train, 'train')
#viewer_train.vis_run()
#config.batch_size = viewer_train.batch_size
viewer_test = Visualizer(config, model, dataset_test, 'test')
viewer_test.vis_run()
def GA_train(population):
for i in range(population.shape[0]):
print("population ", i)
print(population.ix[i])
tf.reset_default_graph()
config, model, dataset_train, dataset_test = argparser(is_train=True)
config.learning_rate_g = population.ix[i]["learning_rate_g"]
config.learning_rate_d = population.ix[i]["learning_rate_d"]
config.update_rate = population.ix[i]["update_rate"]
config.batch_size = population.ix[i]["batch_size"]
config.n_z = population.ix[i]["n_z"]
config.buffer_size = population.ix[i]["buffer_size"]
config.real_probability = population.ix[i]["real_probability"]
model = Model(config, debug_information=config.debug, is_train=True)
trainer = Trainer(config, model, dataset_train, dataset_test)
population.loc[i, "accuracy_score"] = trainer.run_GA()
return population
import sys
import random
import pickle
import numpy as np
import tensorflow as tf
from config import argparser
args = argparser()
with open(args.dataset_dir + 'dataset.pkl', 'rb') as f:
train_set = pickle.load(f, encoding='latin1')
test_set = pickle.load(f, encoding='latin1')
cate_list = pickle.load(f, encoding='latin1')
cate_list = tf.convert_to_tensor(cate_list, dtype=tf.int64)
user_count, item_count, cate_count = pickle.load(f)
class DataLoader:
def __init__(self, batch_size, data):
self.batch_size = batch_size
self.data = data
self.epoch_size = len(self.data) // self.batch_size
if self.epoch_size * self.batch_size < len(self.data):
self.epoch_size += 1
self.i = 0
def __iter__(self):
self.i = 0
return self
else:
trainer.evaluate()
logger.info("Finish evaluating")
def make_log_files(config):
"""
Sets up log directories and saves git diff and command line.
"""
config.run_name = '{}.{}.{}'.format(config.prefix, config.seed,
config.suffix)
config.log_dir = os.path.join(config.log_root_dir, config.run_name)
logger.info('Create log directory: %s', config.log_dir)
os.makedirs(config.log_dir, exist_ok=True)
if config.is_train:
# log config
param_path = os.path.join(config.log_dir, 'params.json')
logger.info('Store parameters in %s', param_path)
with open(param_path, 'w') as fp:
json.dump(config.__dict__, fp, indent=4, sort_keys=True)
if __name__ == '__main__':
args, unparsed = argparser()
if len(unparsed):
logger.error('Unparsed argument is detected:\n%s', unparsed)
else:
run(args)
def debug_test(env, config):
printstar("Debug")
obs = env.reset()
init_car = [
env.car.hull.position[0], env.car.hull.position[1], env.car.hull.angle,
env.car.hull.linearVelocity[0], env.car.hull.linearVelocity[1]
] #, env.info]
obs, reward, dones, info = env.step(1)
for _ in range(100):
obs, reward, dones, info = env.step(1)
env.render()
obs, reward, dones, info = env.step(3)
env.render()
_ = input("Moving state")
print(env._is_outside())
small_network.change_state(env, init_car)
for _ in range(2):
obs, reward, dones, info = env.step(env.action_space.sample())
env.render()
print(env._is_outside())
_ = input("Moved state")
if __name__ == '__main__':
config = argparser()
mp.set_start_method('spawn', force=True)
main(config)
def main(argv=None): # pylint: disable=unused-argument
config = argparser(is_train=True)
all_train_dir, all_result_file_name = construct_train_dir(config)
if config.hdf5FileNametrain == 'train_MRIdata_3_AD_MCI_Normal.hdf5':
dataset_path = os.path.join(r"./datasets/mri/3_AD_MCI_Normal/")
elif config.hdf5FileNametrain == 'train_MRIdata_2_AD_MCI.hdf5':
dataset_path = os.path.join(r"./datasets/mri/2_AD_MCI/")
elif config.hdf5FileNametrain == 'train_MRIdata_2_AD_Normal.hdf5':
dataset_path = os.path.join(r"./datasets/mri/2_AD_Normal/")
else:
dataset_path = os.path.join(r"./datasets/mri/2_MCI_Normal/")
input_file_name = config.hdf5FileNametrain
name_list = input_file_name.split("_")
class_num = name_list[2]
dataset_train_unlabelled, dataset_test, all_hdf5_data_train, all_hdf5_data_test, dataset_train_labelled, dataset_val, all_hdf5_data_val = dataset.create_default_splits8020(
dataset_path, config.hdf5FileNametrain, config.testhdf5FileName,
config.valhdf5FileName, config.idFileNametrain, config.testidFileName,
config.validFileName, config.num_less_label_data, class_num)
data_provider = get_data_provider_by_path(config, dataset_train_unlabelled,
dataset_train_labelled,
dataset_test,
all_hdf5_data_train,
all_hdf5_data_test, dataset_val,
all_hdf5_data_val, 0)
model = TripleGAN3D(config, data_provider, all_train_dir, 0, is_train=True)
if tf.gfile.Exists(all_train_dir[0] + "/GANconfusionMatrixResults"):
log.infov(all_train_dir[0] + "/GANconfusionMatrixResults")
else:
os.makedirs(all_train_dir[0] + "/GANconfusionMatrixResults")
if config.train:
total_start_time = time.time()
print("Data provider train labelled images: ",
data_provider.train_labelled.num_examples)
print("Data provider train unlabelled images: ",
data_provider.train_unlabelled.num_examples)
best_epoch = model.train_all_epochs(config)
total_training_time = time.time() - total_start_time
print(
"\n Total training time for all epoches : %s and %s seconds" %
(str(timedelta(seconds=total_training_time)), total_training_time))
fxx = open(
all_train_dir[0] +
"/GANconfusionMatrixResults/train_timeReport.txt", 'w')
fxx.write(
"\n Total training time for all epoches : %s and %s seconds" %
(str(timedelta(seconds=total_training_time)), total_training_time))
fxx.write('\n')
fxx.close()
if config.test:
model.load_model(best_epoch)
print("Data provider test images: ", data_provider.test.num_examples)
print("Testing...")
total_start_time = time.time()
model.test_and_record(all_result_file_name[0],
0,
config,
all_train_dir[0],
data_provider.test,
batch_size=config.batch_size_label)
total_test_time = time.time() - total_start_time
print("\n Total test time for all epoches : %s and %s seconds" %
(str(timedelta(seconds=total_test_time)), total_test_time))
fxx = open(
all_train_dir[0] +
"/GANconfusionMatrixResults/test_timeReport.txt", 'w')
fxx.write("\n Total test time for all epoches : %s and %s seconds" %
(str(timedelta(seconds=total_test_time)), total_test_time))
fxx.write('\n')
fxx.write(
"\n test time for each record : %s and %s seconds" %
(str(
timedelta(seconds=(total_test_time /
float(data_provider.test.num_examples)))),
(total_test_time / float(data_provider.test.num_examples))))
fxx.write('\n')
fxx.close()
input_file_name = config.hdf5FileNametrain
class_labels = []
name_list = input_file_name.split("_")
if int(name_list[2]) == 3:
class_labels.append(name_list[3])
class_labels.append(name_list[4])
last_class = name_list[5].split(".")
class_labels.append(last_class[0])
else:
class_labels.append(name_list[3])
last_class = name_list[4].split(".")
class_labels.append(last_class[0])
accuracy_10folds_all = []
fold_write = 0
for each_result_file_name in all_result_file_name:
if fold_write < 1:
accuracy, cr, cm, auc = calculateConfusionMatrix(
each_result_file_name, class_labels, all_train_dir[0],
int(name_list[2]))
else:
accuracy, cr, cm, auc = calculateConfusionMatrix(
each_result_file_name, class_labels, './train_dir',
int(name_list[2]))
f = open(
all_train_dir[0] +
"/GANconfusionMatrixResults/ConfusionMatrix.txt", 'w')
log.info("Fold: {}".format(fold_write))
f.write(each_result_file_name)
f.write(
'{}\n\nClassification Report\n\n{}\n\nConfusion Matrix\n\n{}\n'.
format(config.hdf5FileNametrain, cr, cm))
f.write("accuracy: {}\n".format(accuracy))
log.info("accuracy: {}".format(accuracy))
if int(name_list[2]) == 3:
for each_auc in auc:
f.write("auc: {}\n".format(each_auc))
log.info("auc: {}".format(each_auc))
else:
f.write("auc: {}\n".format(auc))
log.info("auc: {}".format(auc))
f.close()
import glob
import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
from models_64x64 import generator as gen
import matplotlib.pyplot as plt
from config import argparser
from data_process import noisy_meas
PARAMS = argparser()
np.random.seed(PARAMS.seed_no)
N = PARAMS.n_mcmc #np.size(mcmc_samps, 0)
burn = int(PARAMS.burn_mcmc*N)
n_eff = N-burn
batch_size = 6400
z_dim = PARAMS.z_dim
n_iter = int(n_eff/batch_size)
dim_like = PARAMS.img_h*PARAMS.img_w*PARAMS.img_c
noise_var = PARAMS.noise_var
img_no = PARAMS.img_no
sample_dir = 'exps/mcmc/img{}_var{}_N{}'.format(PARAMS.img_no, noise_var, N)
mcmc_samps = np.load(sample_dir + '/samples.npy')
eff_samps = np.squeeze(mcmc_samps[burn:,:,:])
plt.figure(figsize=(15, 6))
for ii in range(25):
plt.subplot(5,5,ii+1)
plt.hist(eff_samps[:, ii], 50, density=True);
"git diff >> {}/git.txt".format(config.log_dir),
"echo 'python -m rl.main {}' >> {}/cmd.sh".format(
" ".join([shlex_quote(arg) for arg in sys.argv[1:]]), config.log_dir
),
]
os.system("\n".join(cmds))
# log config
param_path = os.path.join(config.log_dir, "params.json")
logger.info("Store parameters in %s", param_path)
with open(param_path, "w") as fp:
json.dump(config.__dict__, fp, indent=4, sort_keys=True)
if __name__ == "__main__":
parser = argparser()
args, unparsed = parser.parse_known_args()
if "Pusher" in args.env:
from config.pusher import add_arguments
elif "Sawyer" in args.env:
from config.sawyer import add_arguments
else:
raise ValueError("args.env (%s) is not supported" % args.env)
add_arguments(parser)
mp_add_arguments(parser)
args, unparsed = parser.parse_known_args()
if args.debug:
args.rollout_length = 150
