def evaluate_ssd():
"""Evaluate a SSD network."""
# Set results directory and solver
results_dir = osp.join(cfg.OUTPUT_DIR, 'results')
test_solver_file = osp.join(cfg.MODELS_DIR, cfg.DATASET_NAME,
cfg.METHOD_NAME, cfg.MODEL_NAME,
'test_solver.prototxt')
make_if_not_exist(results_dir)
check_if_exist('Solver', test_solver_file)
# Find most recent model
test_model = get_model_path(cfg.OUTPUT_DIR, '.caffemodel', '_iter_')
if test_model is None:
print('No model found in `{:s}`.'.format(cfg.OUTPUT_DIR))
sys.exit()
# Test model
cmd = './frameworks/caffe-rcnn-ssd/build/tools/caffe train \
--solver="{}" --weights="{}" --gpu="{}"\
'.format(test_solver_file, test_model, cfg.GPU_ID)
subprocess.call(cmd, shell=True)
# Set imdb and do evaluation
imdb_name = '{:s}_val'.format(cfg.DATASET_NAME)
imdb = get_imdb(imdb_name)
imdb._do_pascal_voc_eval(results_dir)
def test(data: utils.IMaterialistData, run_config, model=None):
# Load the data
test_data = data.load_test()
# Create the model
if model is None:
model = run_config["model_func"](num_outputs=utils.NUM_LABELS)
model.load_weights(utils.get_model_path(model.run_id), by_name=True)
# Test the model w/ augmentation
predictions = np.zeros((len(test_data), utils.NUM_LABELS))
if run_config["num_test_augment"] == 0:
logging.info("Running testing without augmentation")
run_config["augmenters"] = tuple()
run_config["num_test_augment"] = 1
for _ in range(run_config["num_test_augment"]):
predictions += test_model(model, test_data, **run_config)
predictions /= run_config["num_test_augment"]
# Load the thresholds if we need to
if run_config["threshold"] == "cv":
thresholds = np.load(utils.get_cv_path(model.run_id)).reshape(
(1, predictions.shape[1]))
else:
thresholds = run_config["threshold"]
# Cache the predictions for later
np.save(utils.get_prediction_path(model.run_id), predictions)
data.save_submission(utils.get_submission_path(model.run_id),
predictions,
test_data.ids,
thresholds=thresholds)
def evaluate_faster_rcnn(conf_thresh, nms_thresh):
"""Evaluate a Faster R-CNN network on a image database."""
# Set prototxt
prototxt = osp.join(cfg.MODELS_DIR, cfg.DATASET_NAME, cfg.METHOD_NAME,
cfg.MODEL_NAME, 'test.prototxt')
check_if_exist('Prototxt', prototxt)
# Get most recent model
test_model = get_model_path(cfg.OUTPUT_DIR, '.caffemodel', '_iter_')
if test_model is None:
print('No model found in `{:s}`.'.format(cfg.OUTPUT_DIR))
sys.exit()
caffe.set_mode_gpu()
caffe.set_device(cfg.GPU_ID)
net = caffe.Net(prototxt, caffe.TEST, weights=test_model)
net.name = osp.splitext(osp.basename(test_model))[0]
# Get imdb
imdb_name = '{:s}_val'.format(cfg.DATASET_NAME)
imdb = get_imdb(imdb_name)
# results_dir = osp.join(cfg.OUTPUT_DIR, 'results')
# imdb._do_pascal_voc_eval(results_dir)
if not cfg.TEST.HAS_RPN:
imdb.set_proposal_method(cfg.TEST.PROPOSAL_METHOD)
test_net(net, imdb, conf_thresh, nms_thresh)
def training_loop(dataset, batch_sizes, learning_rates, local_folder, epochs, solver_params,
fit_params, stem='', root='../models', phase_path='', annotator_path=''):
# Training Loop
for batch_size, lr in product(batch_sizes, learning_rates):
# sub path
sub_path = f'{local_folder}/'
if phase_path is not '':
sub_path += f'{phase_path}/'
if annotator_path is not '':
sub_path += f'{annotator_path}/'
# For Documentation
current_time = datetime.datetime.now(pytz.timezone('Europe/Berlin')).strftime("%Y%m%d-%H%M%S")
hyperparams = {'batch': batch_size, 'lr': lr}
writer = get_writer(path=f'../logs/{sub_path}', stem=stem,
current_time=current_time, params=hyperparams)
# Save model path
if local_folder != '' and not os.path.exists('../models/' + sub_path):
os.makedirs('../models/' + sub_path)
path = '../models/'
if local_folder != '':
path += sub_path
save_params = {'stem': stem, 'current_time': current_time, 'hyperparams': hyperparams}
# Training
solver = Solver(dataset, lr, batch_size, writer=writer, save_path_head=path, save_params=save_params,
**solver_params)
model, f1 = solver.fit(**fit_params)
# Save model
model_path = get_model_path(path, stem, current_time, hyperparams, f1)
torch.save(model.state_dict(), model_path + f'_epoch{epochs}.pt')
def train_model(model: SLModel,
trainset: NpDataset,
valset: NpDataset,
epochs=5,
batch_size=32):
# Create the generators
logging.info("Training model for {} epochs and {} batch size".format(
epochs, batch_size))
logging.info("Flowing the train and validation sets")
traingen = trainset.flow(
batch_size=batch_size, shuffle=True, seed=utils.get_random_seed())
valgen = valset.flow(batch_size=batch_size, shuffle=False)
# Create the callbacks
logging.info("Creating the callbacks")
callbacks = [
ModelCheckpoint(
utils.get_model_path(RUN_ID),
"val_loss",
verbose=1,
save_best_only=True),
Plotter(
"loss",
scale='log',
plot_during_train=True,
save_to_file=utils.get_plot_path(RUN_ID),
block_on_end=False),
Plotter(
"accuracy",
scale='linear',
plot_during_train=True,
save_to_file=utils.get_plot_path(RUN_ID + "_acc"),
block_on_end=False)
]
# Create the optiizer
logging.info("Creating the optimizer")
params = [param for param in model.parameters() if param.requires_grad]
# optimizer = optim.SGD(
# params
# lr=0.01,
# momentum=0.9,
# nesterov=True)
optimizer = optim.Adam(params)
logging.info("Optimizer: %r" % optimizer)
# Train the model
logs = model.fit_generator(
traingen,
traingen.steps_per_epoch,
epochs=epochs,
optimizer=optimizer,
validation_generator=valgen,
validation_steps=valgen.steps_per_epoch,
metrics=["accuracy"],
callbacks=callbacks,
verbose=1)
return logs
def _save_model(self, epoch, model, return_f1=False, f1=0.0, early_stopping=False):
if self.save_at is not None and self.save_path_head is not None and self.save_params is not None:
if epoch in self.save_at or early_stopping:
params = self.save_params
if return_f1:
path = get_model_path(
self.save_path_head, params['stem'], params['current_time'], params['hyperparams'], f1)
else:
path = get_model_path(
self.save_path_head, params['stem'], params['current_time'], params['hyperparams'])
path += f'_epoch{epoch}'
if early_stopping:
path += f'_early_stopping'
path += '.pt'
print(f'Saving model at: {path}')
torch.save(model.state_dict(), path)
def cross_validate(data: utils.IMaterialistData, run_config, model=None):
validation_data = data.load_validation_data()
# Create the model
if model is None:
model = run_config["model_func"](num_outputs=utils.NUM_LABELS)
model.load_weights(utils.get_model_path(model.run_id), by_name=True)
return cross_validate_model(model, validation_data, **run_config)
def train(data: HomeCreditData):
train_data = data.load_train()
model = MODEL(input_size=train_data.x.shape[1])
train_data, val_data = train_data.validation_split(
split=0.1, shuffle=True, stratified=True, seed=SPLIT_SEED)
train_model(model, train_data, val_data)
# Load the model and score it
model.load_state(utils.get_model_path(RUN_ID))
score = validate_model(model, val_data)
logging.info("ROC AUC score of best model is {}".format(score))
return model
def predict_with_model(image_path, model_id):
#load model
filepath = utils.get_model_path(model_id)
loaded_model = load_checkpoint_general('checkpoints/' + filepath)
_, classes = predict_top_classes(image_path, loaded_model)
print('classes', classes)
print('element', classes[0])
flowers = get_flowers_name(classes)
return flowers[0]
def test(data: HomeCreditData, model=None):
test_data = data.load_test()
if model is None:
logging.info("No model provided, constructing one.")
model = MODEL(input_size=test_data.x.shape[1])
# Load the model and score it
model.load_state(utils.get_model_path(RUN_ID))
test_preds = test_model(model, test_data)
# Save the submission
data.save_submission(
utils.get_submission_path(RUN_ID),
test_preds,
test_data.ids)
def predict3(image_path):
#load model
results = []
for model_id in utils.get_random_model_ids():
print(model_id)
filepath = utils.get_model_path(model_id)
loaded_model = load_checkpoint_general('checkpoints/' + filepath)
_, classes = predict_top_classes(image_path, loaded_model)
results.append(classes[0])
print(results)
return get_flowers_name(list(utils.results_decider(results)))[0]
def _test_helper_load(dataset_name, log_folder):
# Load pairwise results including node-node matching matrix,
log_folder = join(get_model_path(), 'Our', 'logs', log_folder)
ld = load(join(log_folder, 'final_test_pairs.klepto'))
pairs = ld['test_data_pairs']
print(len(pairs), 'pairs loaded')
# Load graphs.
dataset = load_dataset(dataset_name, 'all', 'mcs', 'bfs') # TODO: check bfs assumption
dataset.print_stats()
natts, *_ = get_dataset_conf(dataset_name)
# node_feat_name = natts[0] if len(natts) >= 1 else None # TODO: only one node feat
from node_feat import encode_node_features
dataset, _ = encode_node_features(dataset)
# TODO: should really load and reset flags but since encode_node_features only uses 'one_hot' it is fine for now
return pairs, dataset
def run_evaluation(exp_name,
run,
ep_no,
inner_cube=(24, 48, 48),
bs=6,
resolution=16):
for mode in ['validation', 'test']:
modelp = utils.get_model_path(exp_name, exp_no=run, ep_no=ep_no)
savep = utils.get_save_path(exp_name,
exp_no=run,
ep_no=ep_no,
mode=mode)
simple_evaluator = Evaluator(modelp, savep,
utils.get_data_path(mode, resolution))
simple_evaluator.run_full_evaluation(inner_cube=inner_cube, bs=bs)
def train_yolov2(no_pretrained=False, resume_training=True):
"""Train a YOLOv2 network."""
data_cfg = osp.join(cfg.OUTPUT_DIR, '{}.data'.format(cfg.DATASET_NAME))
model_cfg = osp.join(cfg.OUTPUT_DIR, '{}.cfg'.format(cfg.DATASET_NAME))
check_if_exist('YOLOv2 data config', data_cfg)
check_if_exist('YOLOv2 model config', model_cfg)
# Set pretrained model
if no_pretrained:
pretrained_model = None
else:
pretrained_model = osp.join(cfg.DATA_DIR, 'imagenet_models',
'{:s}.weights'.format(cfg.MODEL_NAME))
check_if_exist('Pretrained model', pretrained_model)
# Find most recent snapshot
snapshot_file = get_model_path(cfg.OUTPUT_DIR, '.weights', '_batch_')
# Load from most recently saved snapshot, if it exist
if resume_training and snapshot_file != None:
pretrained_model = snapshot_file
snapshot_prefix = cfg.MODEL_NAME + '_' + cfg.METHOD_NAME
# Train model
cmd = ('./frameworks/darknet/darknet-cpp detector train {:s} {:s} {} '
'-gpus {:d} -out {:s}').format(data_cfg, model_cfg,
pretrained_model, cfg.GPU_ID,
snapshot_prefix)
# subprocess.call(cmd, shell=True)
process = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1)
# Log training
try:
with process.stdout, open(cfg.OUTPUT_DIR + '/logfile.txt', 'ab') as f:
for line in iter(process.stdout.readline, b''):
print(line, end='')
f.write(line)
except Exception as e:
print(e)
process.wait()
def train_ssd(no_pretrained, resume_training=True):
"""Train a SSD network."""
train_param = ''
# Set pretrained model
if not no_pretrained:
pretrained_model = osp.join(cfg.DATA_DIR, 'imagenet_models',
'{:s}.caffemodel'.format(cfg.MODEL_NAME))
check_if_exist('Pretrained model', pretrained_model)
train_param = '--weights="{:s}"'.format(pretrained_model)
# Set solver
train_solver_file = osp.join(cfg.MODELS_DIR, cfg.DATASET_NAME,
cfg.METHOD_NAME, cfg.MODEL_NAME,
'train_solver.prototxt')
check_if_exist('Solver', train_solver_file)
# Find most snapshot
snapshot_file = get_model_path(cfg.OUTPUT_DIR, '.solverstate', '_iter_')
# Load from most recently saved snapshot, if it exist
if resume_training and snapshot_file != None:
train_param = '--snapshot="{:s}"'.format(snapshot_file)
# Train model
cmd = './frameworks/caffe-rcnn-ssd/build/tools/caffe train \
--solver="{}" {} --gpu="{}"\
'.format(train_solver_file, train_param, cfg.GPU_ID)
# subprocess.call(cmd, shell=True)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
shell=True)
# Log training
try:
with process.stdout, open(cfg.OUTPUT_DIR + '/logfile.txt', 'ab') as f:
for line in iter(process.stdout.readline, b''):
print(line, end='')
f.write(line)
except Exception as e:
print(e)
process.wait()
def shifted_evaluation(exp_name, run, cp, resolution=16):
for mode in ['validation', 'test']:
modelp = utils.get_model_path(exp_name, exp_no=run, ep_no=cp)
shifted_evaluator = Evaluator(modelp, '',
utils.get_data_path(mode, resolution))
for shift in range(int(shifted_evaluator.sc)):
savep = utils.get_save_path(exp_name,
exp_no=run,
ep_no=cp,
mode=mode,
add='_shift' + str(shift))
shifted_evaluator.reset_save_path(savep)
shifted_evaluator.run_full_evaluation(inner_cube=(24, 48, 48),
bs=6,
safety_margin=(shift,
-shift))
def detect_yolov2(image_paths, result_file, conf_thresh, nms_thresh):
"""Detect object classes in given images with a YOLOv2 network."""
data_cfg = osp.join(cfg.OUTPUT_DIR, '{}.data'.format(cfg.DATASET_NAME))
model_cfg = osp.join(cfg.OUTPUT_DIR, '{}.cfg'.format(cfg.DATASET_NAME))
check_if_exist('YOLOv2 data config', data_cfg)
check_if_exist('YOLOv2 model config', model_cfg)
# Change model config for detection
with open(model_cfg, 'r') as f:
data = f.readlines()
for i in range(len(data)):
if 'height' in data[i]:
data[i] = 'height={:d}\n'.format(cfg.TEST.SCALES[0])
data[i + 1] = 'width={:d}\n'.format(cfg.TEST.MAX_SIZE)
with open(model_cfg, 'w') as f:
f.writelines(data)
# Get model weights
model_weights = get_model_path(cfg.OUTPUT_DIR, '.weights', '_batch_')
if model_weights is None:
print('No model weights found in `{:s}`.'.format(cfg.OUTPUT_DIR))
sys.exit()
# Create temporary list file with image paths
detect_list_file = osp.join(os.getcwd(), 'detect_files.txt')
with open(detect_list_file, "w") as f:
for path in image_paths:
print(path, file=f)
# Add detection list file to data config
with open(data_cfg, "a") as f:
print('detect = {:s}'.format(detect_list_file), file=f)
cmd = ('./frameworks/darknet/darknet-cpp detector detect {} {} {} -out {} '
'-thresh {} -nms_thresh {} -gpus {}').format(
data_cfg, model_cfg, model_weights, result_file, conf_thresh,
nms_thresh, cfg.GPU_ID)
subprocess.call(cmd, shell=True)
# Remove temporary list file with image paths
os.remove(detect_list_file)
def main():
instrument = meta.GUITAR
fx = meta.DISTORTION
param_id = meta.fx_param_ids[0]
model_num = 2
dropout = model_num == 2
unsupervised = True
model_path = get_model_path(instrument, fx, param_id, model_num=model_num)
# Generate input target frame pairs
input_target_pairs = load_test_data(model_path, instrument, fx, param_id)
if unsupervised:
input_target_pairs = [(input_clip, input_clip)
for (input_clip,
target_clip) in input_target_pairs]
data_gen = DataGenerator(input_target_pairs,
floatx=np.float32,
batch_size=meta.params_train['batch'],
frame_size=meta.params_data['frame_size'],
hop_size=meta.params_data['hop_size'],
unsupervised=False)
input_frames, target_frames = data_gen.get_frames()
data_gen_single = DataGenerator([input_target_pairs[0]],
floatx=np.float32,
batch_size=meta.params_train['batch'],
frame_size=meta.params_data['frame_size'],
hop_size=meta.params_data['hop_size'],
unsupervised=False)
input_frame, target_frame = data_gen_single.get_frames()
model = load_model(model_path, dropout=dropout, unsupervised=unsupervised)
preds = model.predict(input_frames) # get model output frames
mae = model.evaluate(input_frames,
target_frames) # get mae over output frames
pred_single = model.predict(input_frame)
print(mae)
print(preds.shape)
np.save(os.path.join(model_path, 'mae.np'), mae)
np.save(os.path.join(model_path, 'preds.np'), preds)
np.save(os.path.join(model_path, 'single_pred.np'), pred_single)
def quant_experiment(exp_name, device, epoch):
config = load_config(exp_name)
config['augmentation'] = False
num_bits = config['num_bits']
net, loss_fn = build_model(config, device, train=False)
model_path, exist_best_model = get_model_path(config, epoch)
assert exist_best_model, "There is no model"
checkpoint = torch.load(model_path, map_location=device)
weights = checkpoint['model_state_dict']
if 'weight_scale' in checkpoint.keys():
weight_scale_list = checkpoint['weight_scale']
print("weight scale loaded")
weights, weight_scale_list = weights_quant_with_scale(
weights, weight_scale_list, num_bits)
print("weight scale completed")
if 'act_scale' in checkpoint.keys():
act_scale_list = checkpoint['act_scale']
print("act scale loaded")
if config['mGPUs']:
net.module.load_state_dict(weights)
else:
net.load_state_dict(weights)
train_loader, val_loader = get_data_loader(config)
# train_metrics = evaluation(config, net, loss_fn, train_loader, device)
# print("------Training Result------")
# print("Prec@1 : ", train_metrics['top1'])
# print("Prec@2 : ", train_metrics['top5'])
# print("Forward Pass Time: ", train_metrics['Forward Pass Time'])
# print("loss : ", train_metrics['loss'])
if 'act_scale' in checkpoint.keys():
val_metrics = quant_evaluation(net, loss_fn, val_loader, device,
act_scale_list, num_bits)
else:
val_metrics = evaluation(config, net, loss_fn, val_loader, device)
print("------Validation Result------")
print("Prec@1 : ", val_metrics['top1'])
print("Prec@5 : ", val_metrics['top5'])
print("Forward Pass Time: ", val_metrics['Forward Pass Time'])
print("loss : ", val_metrics['loss'])
def _load_json_emb(self):
fn = get_save_path() + '/{}_graph2vec_json_dict.pkl'.format(
self.dataset)
if isfile(fn):
with open(fn, 'rb') as handle:
d = load_pkl(handle)
print('Loaded json dict from {}'.format(fn))
return d
dfn = get_model_path(
) + '/graph2vec_tf/embeddings/{}_train_test_dims_{}_epochs_1000_lr_0.3_embeddings.txt'.format(
self.dataset, self.dim)
with open(dfn) as json_data:
d = json.load(json_data)
with open(fn, 'wb') as handle:
save_pkl(d, handle)
print('Loaded json dict from {}\nSaved to {}'.format(dfn, fn))
return d
def evaluate_yolov2(conf_thresh, nms_thresh):
"""Evaluate a YOLOv2 network."""
results_dir = osp.join(cfg.OUTPUT_DIR, 'results')
data_cfg = osp.join(cfg.OUTPUT_DIR, '{}.data'.format(cfg.DATASET_NAME))
model_cfg = osp.join(cfg.OUTPUT_DIR, '{}.cfg'.format(cfg.DATASET_NAME))
make_if_not_exist(results_dir)
check_if_exist('YOLOv2 data config', data_cfg)
check_if_exist('YOLOv2 model config', model_cfg)
# Change model config for testing
with open(model_cfg, 'r') as f:
data = f.readlines()
for i in range(len(data)):
if 'height' in data[i]:
data[i] = 'height={:d}\n'.format(cfg.TEST.SCALES[0])
data[i + 1] = 'width={:d}\n'.format(cfg.TEST.MAX_SIZE)
with open(model_cfg, 'w') as f:
f.writelines(data)
# Find most recent model
test_model = get_model_path(cfg.OUTPUT_DIR, '.weights', '_batch_')
if test_model is None:
print('No model found in `{:s}`.'.format(cfg.OUTPUT_DIR))
sys.exit()
result_file_prefix = '{}_det_test_'.format(cfg.DATASET_NAME)
# Test model
cmd = ('./frameworks/darknet/darknet-cpp detector valid {} {} {} -out {} '
'-gpus {} -nms_thresh {:f}').format(data_cfg, model_cfg, test_model,
result_file_prefix, cfg.GPU_ID,
nms_thresh)
subprocess.call(cmd, shell=True)
# Set imdb and evaluate
imdb_name = '{:s}_val'.format(cfg.DATASET_NAME)
imdb = get_imdb(imdb_name)
imdb._do_pascal_voc_eval(results_dir)
def plot_pred_pairs_from_saved_kelpto(log_folder, dataset_name, num_pairs,
fix_match_pos, want_gid_tuples, need_eps,
mode, pick_best):
log_folder = join(get_model_path(), 'Our', 'logs', log_folder)
ld = load(join(log_folder, 'final_test_pairs.klepto'))
pairs = ld['test_data_pairs']
print(len(pairs), 'pairs loaded')
cnt = 0
pairs_valid = OrderedDict()
for _, pair in pairs.items():
if pair.has_alignment_pred():
cnt += 1
pairs_valid[_] = pair
print('{}/{}={:.2%} has pred matching matrices'.format(
cnt, len(pairs), cnt / len(pairs)))
pairs = pairs_valid
dir = join(log_folder, 'matching_vis')
want = ['true'] if pick_best else ['pred', 'true']
_plot_pairs(pairs, dataset_name, num_pairs, fix_match_pos, dir, want,
want_gid_tuples, need_eps, mode, pick_best)
def rerun_from_loaded_logs(dataset_name, log_folder, theta):
from utils import get_model_path, load
from load_data import load_dataset
from pprint import pprint
print('theta {}'.format(theta))
log_folder = join(get_model_path(), 'Our', 'logs', log_folder)
ld = load(join(log_folder, 'final_test_pairs.klepto'))
pairs = ld['test_data_pairs']
dataset = load_dataset(dataset_name, 'all', 'mcs', 'bfs')
# regenerate y_true_dict_list
for gids in pairs.keys():
gid1, gid2 = gids
g1 = dataset.look_up_graph_by_gid(gid1)
g2 = dataset.look_up_graph_by_gid(gid2)
pair_true = dataset.look_up_pair_by_gids(gid1, gid2)
pair = pairs[gids]
pair.assign_g1_g2(g1, g2)
pair.assign_y_true_dict_list(pair_true.get_y_true_list_dict_view())
# construct flags
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--only_iters_for_debug', type=int, default=None)
parser.add_argument('--dataset', default=dataset_name)
parser.add_argument('--align_metric', default='mcs')
parser.add_argument('--theta', type=float, default=theta)
parser.add_argument('--debug', type=bool, default='debug' in dataset_name)
FLAGS = parser.parse_args()
# call prediction code
pair_list = [pairs[gids] for gids in pairs.keys()]
global_result = eval_pair_list(pair_list, FLAGS)
pprint(global_result)
fn = join(log_folder, 'updated_results_theta_{}.txt'.format(theta))
with open(fn, 'w') as f:
pprint(global_result, stream=f)
def experiment(exp_name, device, epoch):
config = load_config(exp_name)
config['augmentation'] = False
net, loss_fn = build_model(config, device, train=False)
model_path, exist_best_model = get_model_path(config, epoch)
assert exist_best_model, "There is no model"
checkpoint = torch.load(model_path, map_location=device)
if 'model_state_dict' in checkpoint.keys():
weights = checkpoint['model_state_dict']
else:
weights = checkpoint
if config['mGPUs']:
net.module.load_state_dict(weights)
else:
net.load_state_dict(weights)
train_loader, val_loader = get_data_loader(config)
# train_metrics = evaluation(config, net, loss_fn, train_loader, device)
# print("------Training Result------")
# print("Prec@1 : ", train_metrics['top1'])
# print("Prec@2 : ", train_metrics['top5'])
# print("Forward Pass Time: ", train_metrics['Forward Pass Time'])
# print("loss : ", train_metrics['loss'])
val_metrics = evaluation(config, net, loss_fn, val_loader, device)
print("------Validation Result------")
print("Prec@1 : ", val_metrics['top1'])
print("Prec@5 : ", val_metrics['top5'])
print("Forward Pass Time: ", val_metrics['Forward Pass Time'])
print("loss : ", val_metrics['loss'])
if 'model_state_dict' not in checkpoint.keys():
torch.save({
'model_state_dict': weights,
'top1': val_metrics['top1']
}, model_path)
print("model saved at ", model_path)
# Configs
# ===========================================================================
args = args_parse([
('recipe', 'the name of function defined in feature_recipes.py', None),
('-feat', "Acoustic feature", ('mspec', 'mfcc'), 'mspec'),
('-batch', "batch size", None, 64),
('-epoch', "number of epoch", None, 25),
('-l', "audio segmenting length in second", None, 3),
('--debug', "enable debug mode", None, False),
('--train', "force continue training the saved model", None, False),
])
FEAT = args.feat
TRAIN_MODEL = args.train
DEBUG = bool(args.debug)
(EXP_DIR, MODEL_PATH, LOG_PATH,
TRAIN_PATH, TEST_PATH) = get_model_path('xvec', args)
stdio(LOG_PATH)
# ===========================================================================
# Create data feeder
# ===========================================================================
(train, valid,
test_ids, test_dat,
all_speakers) = prepare_dnn_data(
recipe=args.recipe, feat=FEAT, utt_length=args.l)
n_speakers = len(all_speakers) + 1
# ===========================================================================
# Create the network
# ===========================================================================
inputs = [K.placeholder(shape=(None,) + shape[1:],
dtype='float32',
name='input%d' % i)
('recipe', 'the name of function defined in feature_recipes.py', None),
('-nmix', "Number of GMM mixture", None, 2048),
('-tdim', "Dimension of t-matrix", None, 600),
('-feat', "Acoustic feature", ('mspec', 'bnf'), 'bnf'),
('--gmm', "Force re-run training GMM", None, False),
('--stat', "Force re-extraction of centered statistics", None, False),
('--tmat', "Force re-run training Tmatrix", None, False),
('--ivec', "Force re-run extraction of i-vector", None, False),
('--all', "Run all the system again, just a shortcut", None, False),
])
args.gmm |= args.all
args.stat |= args.all | args.gmm
args.tmat |= args.all | args.stat
args.ivec |= args.all | args.tmat
FEAT = args.feat
EXP_DIR, MODEL_PATH, LOG_PATH, TRAIN_PATH, TEST_PATH = get_model_path('ivec', args)
stdio(LOG_PATH)
# ===========================================================================
# Load dataset
# ===========================================================================
X, train, test = prepare_ivec_data(args.recipe, FEAT)
# ===========================================================================
# Training I-vector model
# ===========================================================================
ivec = ml.Ivector(path=MODEL_PATH, nmix=args.nmix, tv_dim=args.tdim,
niter_gmm=16, niter_tmat=16,
downsample=2, stochastic_downsample=True,
device='gpu', name="VoxCelebIvec")
ivec.fit(X, indices=train,
extract_ivecs=True, keep_stats=False)
# ====== extract train i-vector ====== #
for d in [240, 280]:
for s in [48, 64]:
for m in [2, 3, 4]:
for run in range(2):
run_evaluation('FSRCNN_d{0:}_s{1:}_m{2:}'.format(d, s, m),
run, ep_no)
def evaluate_per_saved_epoch(max_epoch,
exp_name,
run,
ep_no,
inner_cube=(24, 48, 48),
bs=6):
for no in range(1, max_epoch):
run_evaluation(exp_name, run, ep_no, inner_cube=inner_cube, bs=bs)
if __name__ == '__main__':
#single_FSRCNN_evaluation()
#FSRCNN_evaluation()
#evaluate_whole_run()
simple_eval = Evaluator(
utils.get_model_path('FSRCNN_d280_s64_m2', 0, 12), 'test_direct.h5',
'/groups/saalfeld/saalfeldlab/larissa/cremi/A_downscaled_times4.crop.h5'
)
simple_eval.run_full_evaluation(inner_cube=(24, 48, 48), bs=6)
#shifted_evaluation('Unet_nl4_nc2_nf64_dc1', 0, 49)
# main()
import os
import cv2
import numpy as np
import utils as u
from detector.landmarks import Landmarker
from detector.transform import Transformer
dirname = os.path.dirname(__file__)
proto = 'deploy.prototxt'
caffe = 'res10_300x300_ssd_iter_140000.caffemodel'
caffePath = u.get_model_path(dirname, caffe)
protoPath = u.get_model_path(dirname, proto)
class Detector:
def __init__(self,
min_conf=0.5,
with_landmarks=False,
with_transformer=False,
colors=None):
self.min_conf = min_conf
self.detector = cv2.dnn.readNetFromCaffe(protoPath, caffePath)
self.landmarker = None if with_landmarks is False else Landmarker()
self.transformer = None if with_transformer is False else Transformer()
self.colors = colors if colors is not None else np.random.uniform(
0, 255, size=(255, 3))
def _verify_landmarker(self):
if self.landmarker is None:
def train_model(model: Model,
train_dataset: ImageDataset,
val_dataset: ImageDataset,
augmenters=(),
epochs=100,
batch_size=32,
epoch_size=10000,
plot=False,
load_model=False,
**kwargs):
logging.info("Training model with run id %s" % model.run_id)
logging.info("Using: \n\tbatch_size: {batch_size} \
\n\tepochs: {epochs} \
\n\tplot: {plot} \
\n\tload_model: {load_model} \
\n\tepoch_size: {epoch_size}".format(**locals()))
if load_model:
logging.info("Reloading model from weights")
model.load_weights(utils.get_model_path(model.run_id), by_name=True)
if model.fine_tune:
old_run_id = model.run_id[:-len("-fine-tune")]
logging.info(
"Fine tuning model with weights from {}".format(old_run_id))
model.load_weights(utils.get_model_path(old_run_id), by_name=True)
steps = epoch_size // batch_size
val_steps = epoch_size // 10 // batch_size
traingen = train_dataset.flow(batch_size=batch_size,
steps_per_epoch=steps,
shuffle=True,
replace=True,
seed=utils.get_random_seed())
valgen = val_dataset.flow(batch_size=batch_size,
steps_per_epoch=val_steps,
shuffle=True,
replace=True,
seed=utils.get_random_seed())
# Add the augmenters to the training generator
for augmenter in augmenters:
traingen = augmenter(traingen)
# Create the callbacks
callbacks = [
ModelCheckpoint(utils.get_model_path(model.run_id),
monitor="val_loss",
save_best_only=False,
save_weights_only=True,
mode="min",
verbose=1),
ModelCheckpoint(utils.get_model_path(model.run_id + "_f1"),
monitor="val_f1_loss",
save_best_only=True,
save_weights_only=True,
mode="min",
verbose=1),
Plotter(monitor="loss",
scale="linear",
plot_during_train=plot,
save_to_file=utils.get_plot_path(model.run_id),
block_on_end=False)
]
# Train the model
history = model.fit_generator(
traingen,
steps_per_epoch=5 if args.debug else traingen.steps_per_epoch,
epochs=epochs,
verbose=1,
callbacks=callbacks,
validation_data=valgen,
validation_steps=5 if args.debug else valgen.steps_per_epoch)
# Log the output
logs = history.history
epochs = range(len(logs["val_loss"]))
checkpoint = min(epochs, key=lambda i: logs["val_loss"][i])
best_val_loss, best_val_f1 = logs["val_loss"][checkpoint], logs[
"val_f1_loss"][checkpoint]
logging.info("LOSS CHECKPOINTED -- Loss: {} -- F1: {}".format(
best_val_loss, best_val_f1))
checkpoint = min(epochs, key=lambda i: logs["val_f1_loss"][i])
best_val_loss, best_val_f1 = logs["val_loss"][checkpoint], logs[
"val_f1_loss"][checkpoint]
logging.info("ACC CHECKPOINTED -- Loss: {} -- F1: {}".format(
best_val_loss, best_val_f1))
def get_model(lang):
if lang not in models:
path = get_model_path(lang)
models[lang] = Word2Vec.load(path)
return models[lang]
def quant_train(exp_name, device, epoch):
num_bits = 8
config = load_config(exp_name)
config['resume_training'] = True
config['resume_from'] = 0
max_epochs = config['max_epochs']
print("make data loader")
train_data_loader, val_data_loader = get_data_loader(config)
net, loss_fn, optimizer, scheduler = build_model(config,
device,
train=True)
ckpt_path, exist_model = get_model_path(config, epoch)
best_top1 = 0
if exist_model:
checkpoint = torch.load(ckpt_path)
if 'top1' in checkpoint.keys():
best_top1 = checkpoint['top1']
print("best top1 score is {:.3f}".format(best_top1))
best_top1 = best_top1 * 0.9 # 90% accuracy
else:
print()
saved_ckpt_path = ckpt_path
checkpoint = torch.load(saved_ckpt_path, map_location=device)
if 'model_state_dict' in checkpoint.keys():
weights = checkpoint['model_state_dict']
else:
weights = checkpoint
if config['mGPUs']:
net.module.load_state_dict(weights)
else:
net.load_state_dict(weights)
print("Successfully loaded trained ckpt at {}".format(saved_ckpt_path))
st_epoch = 0
for g in optimizer.param_groups:
g['lr'] = config['learning_rate']
# quant weight
quant_weights, weight_scale_list = weights_quant(weights,
num_bits=num_bits)
# quant_weights, weight_scale_list = weights_quant_b(net, val_data_loader.dataset, device, weights, num_bits=num_bits)
net.load_state_dict(quant_weights)
for epoch in range(st_epoch, max_epochs):
start_time = time.time()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
net.train()
print("Epoch {}, learning rate : {}".format(
epoch + 1,
scheduler.optimizer.state_dict()['param_groups'][0]['lr']))
# train
start = time.time()
for input, target in tqdm(train_data_loader):
# measure data loading time
data_time.update(time.time() - start)
input_var = input.to(device)
target_var = target.to(device)
if config['model'] == 'tf' or config['model'] == 'tf_fused':
input_var[:, 0, :, :] = (input_var[:, 0, :, :] * 0.229 +
0.485) * 255 - 123.68
input_var[:, 1, :, :] = (input_var[:, 1, :, :] * 0.224 +
0.456) * 255 - 116.78
input_var[:, 2, :, :] = (input_var[:, 2, :, :] * 0.225 +
0.406) * 255 - 103.94
# compute output
output = net(input_var)
if output.shape[1] == 1001:
target_var += 1
loss = loss_fn(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target_var, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
top5.update(prec5[0], input.size(0))
# compute gradient and optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - start)
start = time.time()
scheduler.step()
print("Epoch {}|Time {:.3f}|Training Loss: {:.5f}".format(
epoch + 1,
time.time() - start_time, losses.avg))
print(
"\033[32m training || Prec@1: {:.3f} | Prec@5: {:.3f} \033[37m".
format(top1.avg, top5.avg))
# validation before quantize
tic_val = time.time()
val_metrics = evaluation(config, net, loss_fn, val_data_loader, device)
print("Epoch {}|Time {:.3f}|Validation Loss: {:.5f}".format(
epoch + 1,
time.time() - tic_val, val_metrics['loss']))
print(
"\033[32m validation || Prec@1: {:.3f} | Prec@5: {:.3f} \033[37m".
format(val_metrics['top1'], val_metrics['top5']))
# save recent model
# quant weight
if epoch == 0:
weights = net.state_dict()
quant_weights, weight_scale_list = weights_quant(weights,
num_bits=num_bits)
# quant_weights, weight_scale_list = weights_quant_b(net, val_data_loader.dataset, device, weights, num_bits=num_bits)
net.load_state_dict(quant_weights)
else:
weights = net.state_dict()
# quant_weights, weight_scale_list = weights_quant_with_scale(weights, weight_scale_list, num_bits)
quant_weights, weight_scale_list = weights_quant(weights,
num_bits=num_bits)
net.load_state_dict(quant_weights)
# validation after quantize
tic_val = time.time()
val_metrics = evaluation(config, net, loss_fn, val_data_loader, device)
print("Epoch {}|Time {:.3f}|Validation Loss: {:.5f}".format(
epoch + 1,
time.time() - tic_val, val_metrics['loss']))
print(
"\033[32m validation || Prec@1: {:.3f} | Prec@5: {:.3f} \033[37m".
format(val_metrics['top1'], val_metrics['top5']))
# save best model
if val_metrics['top1'] > best_top1:
if config['mGPUs']:
torch.save(
{
'model_state_dict': net.module.state_dict(),
'top1': val_metrics['top1'],
'weight_scale': weight_scale_list
}, ckpt_path[:-4] + '_quant.pth')
else:
torch.save(
{
'model_state_dict': net.state_dict(),
'top1': val_metrics['top1'],
'weight_scale': weight_scale_list
}, ckpt_path[:-4] + '_quant.pth')
print("\033[32m Best model saved at {}. Prec@1 is {} \033[37m".
format(ckpt_path[:-4] + '_quant.pth', val_metrics['top1']))
best_top1 = val_metrics['top1']
def train(exp_name, device, epoch):
config = load_config(exp_name)
max_epochs = config['max_epochs']
print("make data loader")
train_data_loader, val_data_loader = get_data_loader(config)
net, loss_fn, optimizer, scheduler = build_model(config,
device,
train=True)
best_ckpt_path, exist_best_model = get_model_path(config, "best")
best_top1 = 0
if exist_best_model:
best_checkpoint = torch.load(best_ckpt_path)
best_top1 = best_checkpoint['top1']
print("best top1 score is {:.3f}".format(best_top1))
if config['resume_training']:
saved_ckpt_path = get_model_path(config, "epoch")
checkpoint = torch.load(saved_ckpt_path, map_location=device)
if config['mGPUs']:
net.module.load_state_dict(checkpoint['model_state_dict'])
else:
net.load_state_dict(checkpoint['model_state_dict'])
print("Successfully loaded trained ckpt at {}".format(saved_ckpt_path))
st_epoch = config['resume_from']
else:
st_epoch = 0
for g in optimizer.param_groups:
g['lr'] = config['learning_rate']
for epoch in range(st_epoch, max_epochs):
start_time = time.time()
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
net.train()
print("Epoch {}, learning rate : {}".format(
epoch + 1,
scheduler.optimizer.state_dict()['param_groups'][0]['lr']))
# train
start = time.time()
for input, target in tqdm(train_data_loader):
# measure data loading time
data_time.update(time.time() - start)
input_var = input.to(device)
target_var = target.to(device)
# compute output
output = net(input_var)
loss = loss_fn(output, target_var)
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target_var, topk=(1, 5))
losses.update(loss.item(), input.size(0))
top1.update(prec1[0], input.size(0))
top5.update(prec5[0], input.size(0))
# compute gradient and optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - start)
start = time.time()
scheduler.step()
print("Epoch {}|Time {:.3f}|Training Loss: {:.5f}".format(
epoch + 1,
time.time() - start_time, losses.avg))
print(
"\033[32m training || Prec@1: {:.3f} | Prec@5: {:.3f} \033[37m".
format(top1.avg, top5.avg))
# validation
tic_val = time.time()
val_metrics = evaluation(config, net, loss_fn, val_data_loader, device)
print("Epoch {}|Time {:.3f}|Validation Loss: {:.5f}".format(
epoch + 1,
time.time() - tic_val, val_metrics['loss']))
print(
"\033[32m validation || Prec@1: {:.3f} | Prec@5: {:.3f} \033[37m".
format(val_metrics['top1'], val_metrics['top5']))
# save best model
if val_metrics['top1'] > best_top1:
if config['mGPUs']:
torch.save(
{
'model_state_dict': net.module.state_dict(),
'top1': val_metrics['top1']
}, best_ckpt_path)
else:
torch.save(
{
'model_state_dict': net.state_dict(),
'top1': val_metrics['top1']
}, best_ckpt_path)
print("\033[32m Best model saved at {}. Prec@1 is {} \033[37m".
format(best_ckpt_path, val_metrics['top1']))
best_top1 = val_metrics['top1']
