def main(args_obj):
# parse the config json file
args, _ = get_config_from_json(args_obj.args)
if not os.path.exists(os.path.join(args.exp_log_path, args.exp_name)):
os.makedirs(os.path.join(args.exp_log_path, args.exp_name, 'logs'))
os.makedirs(os.path.join(args.exp_log_path, args.exp_name, 'model'))
os.makedirs(
os.path.join(args.exp_log_path, args.exp_name, 'tensorboard'))
os.makedirs(os.path.join(args.exp_log_path, args.exp_name, 'output'))
args.log_dir = os.path.join(args.exp_log_path, args.exp_name, 'logs')
args.model_dir = os.path.join(args.exp_log_path, args.exp_name, 'model')
args.tensorboard_dir = os.path.join(args.exp_log_path, args.exp_name,
'tensorboard')
args.output_dir = os.path.join(args.exp_log_path, args.exp_name, 'output')
setup_logging(args.log_dir, args.mode)
#Create the Agent and pass all the configuration to it then run it..
agent_class = globals()[args.agent]
agent = agent_class(args)
agent.run()
agent.finalize()
def run_multi():
# Get the arguments
args = get_args()
config, _ = get_config_from_json(args.config)
values_sn = config.exp.vals
values_train = config.exp.vals
values_init = config.exp.vals
params = config.exp.params
section = config.exp.section
# Spectral Normalization
for i in values_sn:
# Mode
for j in values_train:
# Init
for k in values_init:
config[section][params[0]] = i
config[section][params[1]] = j
config[section][params[2]] = k
config.exp.name = args.experiment + "_{}{}{}".format(int(i), int(j), int(k))
process_config(config)
create_dirs(
[
config.log.summary_dir,
config.log.checkpoint_dir,
config.log.step_generation_dir,
config.log.log_file_dir,
config.log.codebase_dir,
]
)
# Copy the model code and the trainer code to the experiment folder
run(config, args)
tf.reset_default_graph()
def main():
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
# except:
# print("missing or invalid arguments")
# exit(0)
if config.gpu_mode is True and not torch.cuda.is_available(
): #虽然开启gpu模式，但是找不到GPU
raise Exception("No GPU found, please run without --gpu_mode=False")
# create an instance of the model you want
# model = Net(config)
model = torch.nn.DataParallel(Net(config), device_ids=[0, 1])
# set the logger
# log_dir = os.path.join(config.save_dir, 'logs_'+config.exp_name)
# if not os.path.exists(log_dir):
# os.makedirs(log_dir)
# logger = Logger(log_dir)
logger = None
train_indices, test_indices = shuffle()
# create your data generator
data_train = DataGenerator(config, 'train').load_dataset()
# create your data generator
data_test = DataGenerator(config, 'test').load_dataset()
# create trainer and pass all the previous components to it
trainer = Trainer(model, config, data_train, logger, data_test)
trainer.train_test()
def process_config_UtsClassification_bayes_optimization(
json_file, learning_rate, model_name, batch_size, num_epochs=100):
config, _ = get_config_from_json(json_file)
config.model.name = model_name
config.model.learning_rate = learning_rate
config.trainer.num_epochs = num_epochs
config.trainer.batch_size = batch_size
config.callbacks.tensorboard_log_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/", time.localtime()),
config.exp.name, config.dataset.name, config.model.name,
"tensorboard_logs", "lr=%s,epoch=%s,batch=%s" %
(config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size))
config.callbacks.checkpoint_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/", time.localtime()),
config.exp.name, config.dataset.name, config.model.name,
"%s-%s-%s" % (config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size), "checkpoints/")
config.log_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/", time.localtime()),
config.exp.name, config.dataset.name, config.model.name,
"%s-%s-%s" % (config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size), "training_logs/")
config.result_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/", time.localtime()),
config.exp.name, config.dataset.name, config.model.name,
"%s-%s-%s" % (config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size), "result/")
return config
def process_config(json_file):
config, _ = get_config_from_json(json_file)
config.summary_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "summary/")
config.img_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "images/")
config_induce = EasyDict(config.config_induce)
config_induce.exp_name = config.exp_name
config_induce.summary_dir = config.summary_dir
config_induce.checkpoint_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "checkpoint/")
config_induce.pseudo_data_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name,
"pseudo_data_dir/")
config_cla = EasyDict(config.config_cla)
config_cla.exp_name = config.exp_name
config_cla.data_numpy_pkl = os.path.join(config_induce.pseudo_data_dir,
"data_numpy.pkl")
return config, config_induce, config_cla
def __init__(self, model_path):
self.config = get_config_from_json('.//config.json')
self.load_path = model_path
self.val_data_dir = self.config.paths.val_data_dir
self.test_data_dir = self.config.paths.test_data_dir
self.model = tf.keras.models.load_model(
self.load_path, custom_objects=get_custom_objects())
print("Model loaded succesfully from " + self.load_path)
def process_config(json_file):
config, _ = get_config_from_json(json_file)
config.summary_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "summary/")
config.checkpoint_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "checkpoint/")
config.pseudo_data_dir = os.path.join(sys.path[0], "../experiments",
config.exp_name, "pseudo_data_dir/")
return config
def main():
# Processing config file
config = get_config_from_json('.\\utils\\config.json')
# Processing data
X_train, Y_train, X_val, Y_val, X_test, Y_test, num_train_features, num_train_samples, num_val_samples, num_test_samples = getData(mypath, config)
svm = SVC(kernel='linear',verbose=1)
print("SVC started")
svm.fit(X_train,Y_train)
print("fit complete")
answer = svm.score(X_test,Y_test)
print("answer:/n",answer)
print("Model completed")
def main():
# Processing config file
config = get_config_from_json('.\\utils\\config.json')
# Processing data
train_dataset, val_dataset, test_dataset, num_train_features, num_train_samples, num_val_samples, num_test_samples = getData(
mypath, config)
# Creating an empty model
dense_model = DenseModel(num_train_features, config)
load_flag = config.experiment.load
# load model from h5 file
if load_flag == True:
try:
print('Loading saved model')
dense_model.load(".\\h5 models\\" + model_version)
results = dense_model.model.evaluate(
test_dataset,
steps=int(num_test_samples / (config.model.batch_size)))
print('test loss, test acc:', results)
except Exception as ex:
print(ex)
print("Invalid model file name provided")
# build and train and save a new model
elif load_flag == False:
try:
dense_model.build_model()
print('Create the trainer')
trainer = Trainer(dense_model.model,
train_dataset,
val_dataset,
config,
steps_per_epoch=int(num_train_samples /
config.model.batch_size),
val_steps=int(num_val_samples /
config.model.batch_size))
print('Start training the model.')
trainer.train()
dense_model.save(".\\h5_models\\" + model_version)
except Exception as ex:
print(ex)
print("Unable to create new model")
else:
print("Invalid load flag in config file")
logging.info('---------Successful execution---------')
def run():
# Get the arguments
args = get_args()
config, _ = get_config_from_json(args.config)
config.exp.name = args.experiment
config = process_config(config)
# create the experiments dirs
create_dirs([
config.log.summary_dir,
config.log.checkpoint_dir,
config.log.step_generation_dir,
config.log.log_file_dir,
config.log.codebase_dir,
])
# Copy the model code and the trainer code to the experiment folder
copy_codebase(config)
l = Logger(config)
logger = l.get_logger(__name__)
# Set the random seed
tf.random.set_random_seed(config.data_loader.random_seed)
# Create the tensorflow session
sess = tf.Session()
# Create the dataloader
data = create("data_loader." + config.data_loader.name)(config)
# Create the model instance
model = create("models.{}.".format(config.data_loader.image_size) +
config.model.name)(config)
# Create the summarizer Object
summarizer = create("utils." + config.log.name)(sess, config)
# Create the trainer
trainer = create("trainers." + config.trainer.name)(sess, model, data,
config, summarizer)
# Load model if exists
model.load(sess)
# Train the model
if args.train:
trainer.train()
# Test the model
if config.trainer.test_at_end:
trainer.test()
logger.info("Experiment has ended.")
def process_config_UtsClassification_bayes_optimization(json_file, params):
config, _ = get_config_from_json(json_file)
config.model.type = params['type']
config.model.convfilt = params['convfilt']
config.model.ksize = params['ksize']
config.model.depth = params['depth']
config.model.drop = params['drop']
config.model.params = params
config.callbacks.tensorboard_log_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/",
time.localtime()), config.exp.name,
config.dataset.name, config.model.name, "tensorboard_logs",
"lr=%s,epoch=%s,batch=%s,type=%s,convfilt=%s,ksize=%s,depth=%s,drop=%s"
% (config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size, config.model.type, config.model.convfilt,
config.model.ksize, config.model.depth, config.model.drop))
config.callbacks.checkpoint_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/",
time.localtime()), config.exp.name,
config.dataset.name, config.model.name, "%s-%s-%s-%s-%s-%s-%s-%s" %
(config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size, config.model.type, config.model.convfilt,
config.model.ksize, config.model.depth, config.model.drop),
"checkpoints/")
config.log_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/",
time.localtime()), config.exp.name,
config.dataset.name, config.model.name, "%s-%s-%s-%s-%s-%s-%s-%s" %
(config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size, config.model.type, config.model.convfilt,
config.model.ksize, config.model.depth, config.model.drop),
"training_logs/")
config.result_dir = os.path.join(
"experiments", time.strftime("%Y-%m-%d/",
time.localtime()), config.exp.name,
config.dataset.name, config.model.name, "%s-%s-%s-%s-%s-%s-%s-%s" %
(config.model.learning_rate, config.trainer.num_epochs,
config.trainer.batch_size, config.model.type, config.model.convfilt,
config.model.ksize, config.model.depth, config.model.drop), "result/")
return config
def myconfig():
args = get_args()
config, _ = get_config_from_json(args.config)
exp_name = config.exp_name
model_name = config.model_name
data_dir = config.data_dir
num_threads = config.num_threads
num_channels = config.num_channels
scale_factor = config.scale_factor
num_epochs = config.num_epochs
save_epochs = config.save_epochs
batch_size = config.batch_size
test_batch_size = config.test_batch_size
save_dir = config.save_dir
lr = config.lr
gpu_mode = config.gpu_mode
load_model= config.load_model
d = 56 # out channels of first layer
s = 32 # out channels of hidden layer
m = 4 # number of layer of hidden layer block
def main():
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
# except:
# print("missing or invalid arguments")
# exit(0)
if config.gpu_mode is True and not torch.cuda.is_available(
): #虽然开启gpu模式，但是找不到GPU
raise Exception("No GPU found, please run without --gpu_mode=False")
# create an instance of the model you want
model = Net(config)
# set the logger
log_dir = os.path.join(config.save_dir, 'logs_' + config.exp_name)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# logger = Logger(log_dir)
logger = None
# create your data generator
# data_train = DataGenerator(config, 'debug').load_dataset()
# create your data generator
data_test = DataGenerator(config, 'test').load_dataset()
data_train = None
# create trainer and pass all the previous components to it
# trainer = Trainer(model, config, data_train, logger, data_test)
# trainer.train_test()
# # create tester and pass all the previous components to it
# 使用最后一个模型：在trainer.py中使用load_model函数
# 使用非最后一个模型：在base_model模块中指定特定模型，并在trainer.py中使用load_spec_model函数
tester = Tester(model, config, data_train, logger, data_test)
with torch.no_grad():
tester.test()
def main():
# capture the config path from the runtime arguments
# then process the json configuration file
args = get_args()
print("Reading config from {}".format(args.config))
config, _ = get_config_from_json(args.config)
# add summary and model directory
config = update_config_by_summary(config)
# if to remove the previous results, set -d 1 as a parameter
print('Whether to del the previous saved model', args.delete)
if args.delete == '1':
# delete existing checkpoints and summaries
print('Deleting existing models and logs from:')
print(config.summary_dir, config.checkpoint_dir)
remove_dir(config.summary_dir)
remove_dir(config.checkpoint_dir)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
"""Load data"""
# load global word, position and tag vocabularies
word_vocab = load_vocab(path=config.datadir + config.word_vocab_path,
mode='word')
position_vocab = load_vocab(path=config.datadir + config.pos_vocab_path,
mode='pos')
tag_vocab = load_vocab(path=config.datadir + config.tag_vocab_path,
mode='tag')
config.word_vocab_size = len(word_vocab)
config.pos_vocab_size = len(position_vocab)
config.tag_vocab_size = len(tag_vocab)
print('word vocab size:', config.word_vocab_size)
# create your data generator to load train data
x_path = config.datadir + config.train_path
train_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
train_loader.load_data()
# update the max length for each patient and each visit to be used in lstm
train_max_patient_len = train_loader.max_patient_len
train_max_visit_len = train_loader.max_visit_len
# create your data generator to load valid data
x_path = config.datadir + config.valid_path
valid_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
valid_loader.load_data()
valid_max_patient_len = valid_loader.max_patient_len
valid_max_visit_len = valid_loader.max_visit_len
# create your data generator to load test data
x_path = config.datadir + config.test_path
test_loader = DataLoader(config, x_path, word_vocab, position_vocab,
tag_vocab)
test_loader.load_data()
test_max_patient_len = test_loader.max_patient_len
test_max_visit_len = test_loader.max_visit_len
print("The max patient lengths for train, valid and test are {}, {}, {}".
format(train_max_patient_len, valid_max_patient_len,
test_max_patient_len))
print("The max visit lengths for train, valid and test are {}, {}, {}".
format(train_max_visit_len, valid_max_visit_len, test_max_visit_len))
# select the maximum lengths of visits and codes as the size of lstm
config.max_patient_len = max(
[train_max_patient_len, valid_max_patient_len, test_max_patient_len])
config.max_visit_len = max(
[train_max_visit_len, valid_max_visit_len, test_max_visit_len])
train_loader.pad_data(config.max_patient_len, config.max_visit_len)
valid_loader.pad_data(config.max_patient_len, config.max_visit_len)
test_loader.pad_data(config.max_patient_len, config.max_visit_len)
# add num_iter_per_epoch to config for trainer
config.train_size = train_loader.get_datasize()
config.valid_size = valid_loader.get_datasize()
config.test_size = test_loader.get_datasize()
config.num_iter_per_epoch = int(config.train_size / config.batch_size)
print("The sizes for train, valid and test are {}, {}, {}".format(
config.train_size, config.valid_size, config.test_size))
"""Run model"""
# create tensorflow session
# specify only using one GPU
tfconfig = tf.ConfigProto(device_count={'GPU': 1})
# allow the dynamic increase of GPU memory
tfconfig.gpu_options.allow_growth = True
# limit the maximum of GPU usage as 0.5
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.5
with tf.Session(config=tfconfig) as sess:
# create an instance of the model you want
model = Model(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = PredTrainer(sess, model, train_loader, valid_loader,
test_loader, config, logger)
# load model if exists
model.load(sess)
# here you train your model
trainer.train()
def main():
os.environ['CUDA_VISIBLE_DEVICES'] = '2'
# capture the config path from the run arguments
# then process the json configuration file
# try:
args = get_args()
config, _ = get_config_from_json(args.config)
bayes_config = {
"algorithm": "bayes",
"parameters": {
# "model": {"type": "categorical", "values": ['cnn','mlp']},
"learning_rate": {
"type": "float",
"min": 0.001,
"max": 0.01
},
# "batch_size": {"type": "integer", "min": 16, "max": 32},
# "num_epochs": {"type": "integer", "min": 5, "max": 10},
},
"spec": {
"maxCombo": 10,
"objective": "minimize",
"metric": "test_f1",
"minSampleSize": 100,
"retryAssignLimit": 0,
},
"trials": 1,
"name": "Bayes",
}
opt = Optimizer(bayes_config,
api_key=config.comet_api_key,
project_name=config.exp_name)
for exp in opt.get_experiments():
args = get_args()
# config = process_config_UtsClassification_bayes_optimization(args.config, exp.get_parameter('model'),exp.get_parameter('learning_rate'),
# exp.get_parameter('batch_size'), exp.get_parameter('num_epochs'))
config = process_config_UtsClassification_bayes_optimization(
args.config, exp.get_parameter('learning_rate'))
# except:
# print("missing or invalid arguments")
# exit(0)
# create the experiments dirs
print('Create the data generator.')
data_loader = UtsClassificationDataLoader(config)
print('Create the model.')
model = UtsClassificationModel(config, data_loader.get_inputshape(),
data_loader.get_nbclasses())
print('Create the trainer')
trainer = UtsClassificationTrainer(model.model,
data_loader.get_train_data(),
config)
print('Start training the model.')
trainer.train()
# print('Create the evaluater.')
# evaluater = UtsClassificationEvaluater(trainer.best_model, data_loader.get_test_data(), data_loader.get_nbclasses(),
# config)
#
# print('Start evaluating the model.')
# evaluater.evluate()
exp.log_metric("test_f1", trainer.best_model_val_loss)
print('done')
def main():
detector_config = get_config_from_json('configs/orb_config.json')
drone_config = get_config_from_json('configs/drone_config.json')
glob_config = get_config_from_json('configs/glob_config.json')
landscape_map = Image.read(glob_config.path_to_map)
detector = HashPointDetector(detector_config)
drone = Drone(drone_config, glob_config, detector, landscape_map)
drone.load_hashes(detector.config.path_to_hashes)
visualizer = Visualizer(landscape_map)
visualizer.draw_trajectory(xsmooth, ysmooth)
position = drone.get_position()
shift = random.random() * 100.0
count_step = 0
for node in nodes:
destination = node['point']
detect_position = drone.get_position_from_image()
while drone.get_distance_to_point(destination) > 32:
prev_position = drone.get_position()
visualizer.draw_line_moving(prev_position, position)
position = drone.get_position()
if count_step % 4 == 0:
print('Take picture')
position = drone.get_position()
start_time = time.time()
detect_position = drone.get_position_from_image()
mse_value = mse(position, detect_position)
detect_real_errors.append(mse_value)
visualizer.dashboard_text(4, 'MSE: {}'.format(mse_value))
time_detecting = time.time() - start_time
visualizer.dashboard_text(5, 'Time: {}'.format(time_detecting))
times_detecting.append(time_detecting)
visualizer.init_debug_frame()
visualizer.draw_destination(destination)
picture_params = drone.get_picture_params()
visualizer.update_landscape_debug_frame()
drone.move(destination)
visualizer.draw_drone_moving(picture_params, position,
drone.real_pos, detect_position)
visualizer.draw_drone_picture(drone.get_picture())
visualizer.dashboard_text(
0, 'Speed: {}'.format(float(drone.get_speed())))
visualizer.dashboard_text(
1, 'Height: {}'.format(
200.0 + 15 * math.sin(shift + time.time() * 0.017) +
7 * math.cos(shift * 13 + time.time() * 0.15)))
visualizer.dashboard_text(
2, 'Angle: {}'.format(math.degrees(abs(drone.get_rotation()))))
visualizer.dashboard_text(
3, 'Keypoints: {}'.format(drone.cur_candidates))
visualizer.dashboard_text(
6, 'Average MSE: {}'.format(median(detect_real_errors)))
visualizer.dashboard_text(
7, 'Average time: {}'.format(median(times_detecting)))
count_step += 1
cv2.waitKey(50)
print('Detect-real MSE: ',
sum(detect_real_errors) / len(detect_real_errors))
print('Mean time detecting: ', sum(times_detecting) / len(times_detecting))
visualizer.wait()
import tensorflow as tf
from utils.config import get_args
from utils.config import get_config_from_json
from utils.utils import judge_and_new
import os
if __name__ == '__main__':
# tf.app.run()
''' Dynamic configs '''
try:
args = get_args()
config = get_config_from_json(args.config)
except:
raise Exception("missing or invalid arguments")
''' Static configs '''
os.environ["CUDA_VISIBLE_DEVICES"] = "0, 2"
judge_and_new(os.path.join(config.work_root, config.project_name))
config.ckpt_path = os.path.join(config.work_root, config.project_name,
'save')
judge_and_new(os.path.join(config.work_root, config.project_name, 'save'))
# config.timeline_path = os.path.join(config.work_root, config.project_name, 'timelines')
# judge_clean_new(os.path.join(config.work_root, config.project_name, 'timelines'))
config.tensorb_path = os.path.join(config.work_root, config.project_name,
'tensorb')
judge_and_new(
os.path.join(config.work_root, config.project_name, 'tensorb'))
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.85,
allow_growth=True)
# gpu_options = tf.GPUOptions(allow_growth=True)
def main():
# capture the config path from the run arguments
# then process the json configuration file
# args = get_args()
config = process_config(args.config_folder+"train_config.json")
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create your data generator
cfg = get_config_from_json(args.config_folder+"data_config.json")[0]
data = DataGenerator(cfg)
IMAGENET_FILEPATH = os.path.join(cfg.BASE_PATH, cfg.IMAGENET_FILEPATH)
MODEL_FILEPATH = os.path.join(cfg.BASE_PATH, cfg.MODEL_FILEPATH)
# create novelty GAN
novelty_gan = NoveltyGAN(generator_output_classes=cfg.OUTPUT_CLASSES, fcn=True, upsampling=False, alpha=0.25,
imagenet_filepath=IMAGENET_FILEPATH, model_filepath=MODEL_FILEPATH)
# create the trainer object
trainer = NoveltyGANTrainer(novelty_gan, data, config)
# loss = trainer.train_step_gan()
# loss = trainer.train_step(train_on_real_data=True)
# print("loss = ", loss)
for epoch_id in range(config.num_epochs):
loss = trainer.train_epoch(epoch_id)
# novelty_gan.gan.summary()
# Sanity check: works.
"""
for batch in data.next_batch(3):
labels_batch = novelty_gan.generator.predict_on_batch(batch[0])
print("Image from batch:", batch[0].shape)
print("Ground truth labels:", batch[1].shape)
print("Predicted labels:", labels_batch.shape)
# isReal = novelty_gan.discriminator.predict_on_batch([batch[1], batch[0]])
# isReal = novelty_gan.discriminator.predict_on_batch([labels_batch, batch[0]])
isReal = novelty_gan.gan.predict_on_batch(batch[0])
print("discriminator prediction for ground-truth:", isReal.shape)
"""
# Yet another sanity check for training.
"""
for batch in data.next_batch(3):
img_batch = batch[0]
labels_batch = batch[1]
# predicted_labels_batch = novelty_gan.generator.predict_on_batch(img_batch)
target = np.zeros((3, 2))
target[:, 0] = 1
loss = novelty_gan.discriminator.train_on_batch([labels_batch, img_batch], target)
print(loss)
"""
if 0:
print('_________________')
print(cfg)
print('_________________')
def process_config(json_file):
config, _ = get_config_from_json(json_file)
return config
from metrics.metrics import BinaryMetrics, MulticlassMetrics
from sklearn.preprocessing import LabelBinarizer
from utils.config import get_config_from_json
from pathlib import Path
import numpy as np
import os
config, _ = get_config_from_json('configs/config_metrics.json')
results, _ = get_config_from_json(config.results.file)
n_classes = len(config.dataset.classes)
prob_predictions = np.array(results.prob_predictions)
ground_truth = np.array(results.ground_truth)
categorical_ground_truth = ground_truth
n_samples = ground_truth.size
if( len(ground_truth.shape) == 1):
# prob_predictions_1d = prob_predictions
# prob_predictions = np.zeros((n_samples, ground_truth.max()+1), dtype=float)
# prob_predictions[np.arange(n_samples), 0] = 1 - prob_predictions_1d[np.arange(n_samples)]
# prob_predictions[np.arange(n_samples), 1] = prob_predictions_1d[np.arange(n_samples)]
predictions = np.round(prob_predictions + 0.01).astype(int)
categorical_ground_truth = np.zeros((n_samples, ground_truth.max()+1), dtype=int)
categorical_ground_truth[np.arange(n_samples), ground_truth] = 1
elif( len(ground_truth.shape) == 2):
predictions = np.argmax(prob_predictions, axis=-1)
ground_truth = np.argmax(categorical_ground_truth, axis=-1)
if(n_classes == 2):
def test_get_config_from_json(self):
from utils.config import get_config_from_json
json_file = '/home/cl3720/2019-concept2vec/deep-learning-skeleton/./experiments/configs.json'
config, _ = get_config_from_json(json_file)
self.assertTrue('data' in config)
index = 0
with Path(path).open('r') as f:
for l in f:
key = l.strip() if mode == 'word' else int(l.strip())
vocab[key] = index
index += 1
return vocab
if __name__ == '__main__':
"""
load data and build vocab
"""
# load data from disk
config_path = '../configs/text_ner.json'
config = get_config_from_json(config_path)
datadir = '../data_samples/ner/'
words_path = datadir + 'corpus_words.txt'
tags_path = datadir + 'corpus_tags.txt'
X, Y = [], []
with Path(words_path).open('rb') as f:
for l in f:
X.append(l.strip().split())
word_vocab = build_vocab(X)
with Path(tags_path).open('rb') as f:
for l in f:
Y.append(l.strip().split())
tag_vocab = build_vocab(Y)
# add padding token
if PAD_WORD not in word_vocab: word_vocab.add(PAD_WORD)
if PAD_TAG not in tag_vocab: tag_vocab.add(PAD_TAG)
for tile in tiles:
dx, dy, _, _ = tile['coordinates']
kp, des = detector.create_features(tile['image_cls'].img)
for key_point in kp:
key_point.pt = (key_point.pt[0] + dx, key_point.pt[1] + dy)
image_bin = binImages(kp, des)
hashes = get_hashes(image_bin)
result.append({
'tile_coordinates': tile['coordinates'],
'hashes': hashes
})
return result
if __name__ == '__main__':
detector_config = get_config_from_json('configs/orb_config.json')
glob_config = get_config_from_json('configs/glob_config.json')
landscape_map = Image.read(glob_config.path_to_map)
detector = HashPointDetector(detector_config)
tiles = landscape_map.get_tiles(tile_size=glob_config.tile_size)
landscape_hashes = create_hashes(tiles, detector)
with open(detector.config.path_to_hashes, 'wb') as des_file:
pickle.dump(landscape_hashes, des_file)
print('Done!')
# def create_hashes(tiles, detector):
# result = []
# for tile in tiles:
# dx, dy, _, _ = tile['coordinates']
# sub_tiles = tile['image_cls'].get_tiles(tile_size=(
# glob_config.tile_size[0] // 2,
from dataloaders.dataloader import DataLoader
from models.conv_model import ConvModel
from trainers.conv_model_trainer import ConvModelTrainer
from utils.config import get_config_from_json
from utils.args import get_args
if __name__ == "__main__":
args = get_args() # parse args
config, _ = get_config_from_json(args.config) # load config
try:
args = get_args() #parse args
config, _ = get_config_from_json(args.config) #load config
except FileNotFoundError:
print("File {} don't exists".format(args.config))
exit(0)
except Exception:
print(("Missing or invalid arguments"))
exit(0)
dataloader = DataLoader('datasets/data', config) #create data_loader
train_data, valid_data = dataloader.create_datasets()
model = ConvModel(config)
trainer = ConvModelTrainer(config, model)
trainer.train(train_data, valid_data)
def __init__(self, model_path):
self.config = get_config_from_json('.//config.json')
self.load_path = model_path
self.vocab_path = self.config.paths.vocab_path
self.val_data_dir = self.config.paths.val_data_dir
self.test_data_dir = self.config.paths.test_data_dir
def __init__(self):
self.config = get_config_from_json('.//config.json')
self.callbacks = []
self.init_callbacks()
self.train()
