def main():
# DATA PREPARATION
dataset = pd.read_csv('dataset.csv', encoding="ISO-8859-1")
X_train, X_val, X_test, y_train, y_val, y_test, vect = prepare_dataset(
dataset)
# MODELS TRAINING
print("\n--------------------------------------------------------")
print("------------------- MODELS TRAINING -------------------")
print("--------------------------------------------------------\n")
# Random Forest
modelRF = train_RF(X_train, y_train)
eval_model(modelRF, X_val, y_val)
# SVM
modelSVC = train_SVC(X_train, y_train)
eval_model(modelSVC, X_val, y_val)
#NN
modelNN = train_NN(X_train, y_train)
eval_model(modelNN, X_val, y_val)
# MODELS TESTING
print("\n--------------------------------------------------------")
print("-------------- MODELS TESTING (accuracy) --------------")
print("--------------------------------------------------------\n")
print("RANDOM FOREST: ", test_model(modelRF, X_test, y_test))
print("SVC: ", test_model(modelSVC, X_test, y_test))
print("NEURAL NETWORK: ", test_model(modelNN, X_test, y_test))
# RUN THE DEMO
run_demo(vect, modelNN)
def test_all_models(dev, test, glove, folder = 'models/'):
files = os.listdir(folder)
extless = set([file.split('.')[0] for file in files if os.path.isfile(file)]) - set([''])
epoch_less = set([file.split('~')[0] for file in extless])
for model_short in epoch_less:
if model_short in extless:
modelname = model_short
else:
same_exper = [m for m in extless if m.startswith(model_short)]
epoch_max = max([int(file.split('~')[1]) for file in same_exper])
modelname = model_short + '~' + str(epoch_max)
print modelname
model = models.load_model(folder + modelname)
dev_acc = models.test_model(model, dev, glove)
test_acc = models.test_model(model, test, glove)
print "Dev:", '{0:.2f}'.format(dev_acc * 100), "Test_acc:", '{0:.2f}'.format(test_acc * 100)
print
def test_tautologies(train, dev, glove, paths = ['aug0','aug005','aug015','aug05']):
testsets = [dev, generate_tautologies(dev), generate_contradictions(dev), generate_neutral(dev)]
names = ['dev' , 'ent', 'contr' ,'neu']
for path in paths:
print path
model_path = misc.best_model_path('models/' + path)
model = models.load_model(model_path)
accs = [models.test_model(model, dataset, glove) for dataset in testsets]
for name, dataset, acc in zip (names, testsets, accs):
print name, acc, len(dataset)
def process_rdd(train_rdd):
# Append the preprocessed rdd to the global list.
RDD_LIST.append(ec_spark_context.parallelize(train_rdd.collect()))
rddlist_accuracy_rdd = list()
global_error = 1
if len(RDD_LIST) >= MINIMUM_RDD_LIST_LEN:
for i in range(0, min(K, MINIMUM_RDD_LIST_LEN)):
empty_rdd = ec_spark_context.parallelize([])
rdd_subset = get_k_rdds_from_list(RDD_LIST, k=K)
rdd = get_merged_rdd(empty_rdd, rdd_subset)
min_error = min(
list(
map(lambda model: test_model(model, poker_hands_test_rdd),
precreate_models(rdd))))
rddlist_accuracy_rdd.append((rdd_subset, min_error))
global_error = min(global_error, min_error)
print("Error: " + str(global_error))
if len(RDD_LIST) > MINIMUM_RDD_LIST_LEN:
remove_least_quality_rdd(RDD_LIST,
rddlist_accuracy_rdd,
minimum_list_len=MINIMUM_RDD_LIST_LEN,
optimizing_fn=max)
# generate and divide data
x_all, y_all, y_train, y_val, y_test, val_mask, train_mask, test_mask = generate_samples()
# generate support
num_supports = 1 + FLAGS.max_degree
support = get_normal_support(y_train, num_supports)
# init place holders and model
placeholders = {
# 'features': tf.sparse_placeholder(tf.float32, shape=tf.constant(features[2], dtype=tf.int64)),
'support': [tf.placeholder(tf.float32) for _ in range(num_supports)],
'labels': tf.placeholder(tf.float32, shape=(None, 1)),
'labels_mask': tf.placeholder(tf.int32),
'dropout': tf.placeholder_with_default(0.5, shape=()),
}
model = test_model(placeholders, input_dim=1)
# Initialize session
sess = tf.Session()
sess.run(tf.global_variables_initializer())
cost_val = []
# Train model
for epoch in range(200):
t = time.time()
# Construct feed dictionary
feed_dict = construct_feed_dict(support, y_train, train_mask, placeholders)
feed_dict.update({placeholders['dropout']: 0.5})
# Training step
import models, utils, datasets, predict
import logging, sys
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
scales = {'Chicago': (0.78, 0.88, 1),
'Chicago RAC': (0.6, 0.8, 0.1),
'New Haven': (0.89, 1, 0.68),
'New Haven RAC': (1, 1, 1),
'Oakland': (0.84, 1, 0.51),
'Oakland RAC': (1, 0.94, 0.23),
'Richmond': (0.64, 1, 1),
'Richmond RAC': (1, 1, 1)}
if __name__=='__main__':
if len(sys.argv) == 1:
print "No model name was given! Run again using format: \n\t",
print "python test.py modelname"
else:
modelname = sys.argv[1]
pred = models.test_model(modelname)
categories = datasets.load_dataset('Categories')
n_pred = pred.shape[0]
pred = predict.apply_scales(pred, categories[-n_pred:], scales)
name = modelname + ".csv"
utils.create_submission(name, pred)
print "Saved submission with name %s" %(name)
print("Image Super-resolution task\n")
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
np.warnings.filterwarnings('ignore')
###############################################################
# 1 Produce .pb model file that will be used for validation #
###############################################################
print("Saving pre-trained model as .pb file")
g = tf.Graph()
with g.as_default(), tf.Session() as sess:
image_ = tf.placeholder(tf.float32, shape=(1, None, None, 3), name="input")
out_ = tf.identity(test_model(image_), name="output")
saver = tf.train.Saver()
saver.restore(sess, model_location)
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, g.as_graph_def(), "input,output".split(",")
)
tf.train.write_graph(output_graph_def, 'models_converted', 'model.pb', as_text=False)
print("Model was successfully saved!")
print("\n-------------------------------------\n")
sess.close()
def main(args, cfg):
logging.info(f"=========> {cfg.EXP} <=========")
# Declare variables
start_epoch = 0
best_metric = 0.
# Create model
model = get_model(cfg)
# Define Loss and Optimizer
train_criterion = nn.CrossEntropyLoss()
valid_criterion = nn.CrossEntropyLoss()
optimizer = optim.AdamW(params=model.parameters(),
lr=cfg.OPT.BASE_LR,
weight_decay=cfg.OPT.WEIGHT_DECAY)
# CUDA & Mixed Precision
if cfg.SYSTEM.CUDA:
model = model.cuda()
train_criterion = train_criterion.cuda()
valid_criterion = valid_criterion.cuda()
if cfg.SYSTEM.FP16:
model, optimizer = amp.initialize(
models=model,
optimizers=optimizer,
opt_level=cfg.SYSTEM.OPT_L,
keep_batchnorm_fp32=(True if cfg.SYSTEM.OPT_L == "O2" else None))
# Load checkpoint
if args.load != "":
if os.path.isfile(args.load):
print(f"=> loading checkpoint {args.load}")
ckpt = torch.load(args.load, "cpu")
model.load_state_dict(ckpt.pop('state_dict'))
if not args.finetune:
print("resuming optimizer ...")
optimizer.load_state_dict(ckpt.pop('optimizer'))
start_epoch, best_metric = ckpt['epoch'], ckpt['best_metric']
logging.info(
f"=> loaded checkpoint '{args.load}' (epoch {ckpt['epoch']}, best_metric: {ckpt['best_metric']})"
)
else:
logging.info(f"=> no checkpoint found at '{args.load}'")
if cfg.SYSTEM.MULTI_GPU:
model = nn.DataParallel(model)
# Load data
DataSet = CifarDS
train_ds = DataSet(cfg, mode="train")
valid_ds = DataSet(cfg, mode="valid")
test_ds = DataSet(cfg, mode="test")
# Dataloader
if cfg.DEBUG:
train_ds = Subset(train_ds,
np.random.choice(np.arange(len(train_ds)), 20))
valid_ds = Subset(valid_ds,
np.random.choice(np.arange(len(valid_ds)), 10))
train_loader = DataLoader(train_ds,
cfg.TRAIN.BATCH_SIZE,
pin_memory=False,
shuffle=True,
drop_last=False,
num_workers=cfg.SYSTEM.NUM_WORKERS)
valid_loader = DataLoader(valid_ds,
cfg.TRAIN.BATCH_SIZE,
pin_memory=False,
shuffle=False,
drop_last=False,
num_workers=cfg.SYSTEM.NUM_WORKERS)
test_loader = DataLoader(test_ds,
cfg.TRAIN.BATCH_SIZE,
pin_memory=False,
shuffle=False,
drop_last=False,
num_workers=cfg.SYSTEM.NUM_WORKERS)
scheduler = LR_Scheduler("cos", cfg.OPT.BASE_LR, cfg.TRAIN.EPOCHS,\
iters_per_epoch=len(train_loader),
warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
if args.mode == "train":
train_loop(logging.info, cfg, model, \
train_loader, train_criterion, valid_loader, valid_criterion, \
optimizer, scheduler, start_epoch, best_metric)
elif args.mode == "valid":
valid_model(logging.info,
cfg,
model,
valid_loader,
valid_criterion,
tta=cfg.INFER.TTA)
else:
test_model(logging.info, cfg, model, test_loader, tta=cfg.INFER.TTA)
results.append(CHAR_TO_ID[letter])
else:
results.append(CHAR_TO_ID['-'])
return results
def token_to_phone(seq):
results = []
for val in seq:
if val in ID_TO_PHONE:
results.append(ID_TO_PHONE[val])
else:
results.append('~')
return results
word = 'TENSOR'
tokens = word_to_token(word)
print(tokens)
word = ([tokens], len(tokens))
with tf.Graph().as_default() as g:
logits = test_model([tokens], len(tokens), 1)
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, "../models/models.ckpt")
predictions = sess.run(logits)
print(predictions)
print(token_to_phone(predictions[0]))
def main(args, cfg):
logging.info(f"=========> {cfg.EXP} <=========")
#tensorboard
# tb = SummaryWriter(f"runs/{cfg.EXP}", comment=f"{cfg.COMMENT}") #for visualization
# Declare variables
start_epoch = 0
best_metric = 0.
# Create model
model = get_model(cfg)
# Define Loss and Optimizer
# train
# train_criterion = SoftDiceLoss()
# valid_criterion = SoftDiceLoss()
# train_criterion = BinaryDiceLoss()
# valid_criterion = BinaryDiceLoss()
train_criterion = nn.CrossEntropyLoss()
valid_criterion = nn.CrossEntropyLoss()
# train_criterion = nn.BCEWithLogitsLoss()
# valid_criterion = nn.BCEWithLogitsLoss()
# train_criterion = get_loss(cfg)
# valid_criterion = get_loss(cfg)
# #optimizer
optimizer = optim.AdamW(params=model.parameters(),
lr=cfg.OPT.BASE_LR,
weight_decay=cfg.OPT.WEIGHT_DECAY)
# optimizer = optim.Adadelta(params=model.parameters(),
# lr=cfg.OPT.BASE_LR,
# weight_decay=cfg.OPT.WEIGHT_DECAY)
# CUDA & Mixed Precision
if cfg.SYSTEM.CUDA:
model = model.cuda()
train_criterion = train_criterion.cuda()
valid_criterion = valid_criterion.cuda()
if cfg.SYSTEM.FP16:
model, optimizer = amp.initialize(
models=model,
optimizers=optimizer,
opt_level=cfg.SYSTEM.OPT_L,
keep_batchnorm_fp32=(True if cfg.SYSTEM.OPT_L == "O2" else None))
# Load checkpoint
if args.load != "":
if os.path.isfile(args.load):
print(f"=> loading checkpoint {args.load}")
ckpt = torch.load(args.load, "cpu")
model.load_state_dict(ckpt.pop('state_dict'))
if not args.finetune:
print("resuming optimizer ...")
optimizer.load_state_dict(ckpt.pop('optimizer'))
start_epoch, best_metric = ckpt['epoch'], ckpt['best_metric']
logging.info(
f"=> loaded checkpoint '{args.load}' (epoch {ckpt['epoch']}, best_metric: {ckpt['best_metric']})"
)
else:
logging.info(f"=> no checkpoint found at '{args.load}'")
if cfg.SYSTEM.MULTI_GPU:
model = nn.DataParallel(model)
# Load data
train_loader = get_dataset('train', cfg)
valid_loader = get_dataset('valid', cfg)
test_loader = get_test('valid', cfg)
if cfg.DEBUG:
train_loader = get_debug_dataset('train', cfg)
valid_loader = get_debug_dataset('valid', cfg)
scheduler = LR_Scheduler("cos", cfg.OPT.BASE_LR, cfg.TRAIN.EPOCHS,\
iters_per_epoch=len(train_loader),
warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
if args.mode == "train":
train_loop(logging.info, cfg, model, \
train_loader, valid_loader, train_criterion, valid_criterion,\
optimizer, scheduler, start_epoch, best_metric)
elif args.mode == "valid":
valid_model(logging.info,
cfg,
model,
valid_criterion,
valid_loader,
tta=cfg.INFER.TTA)
else:
test_model(logging.info,
cfg,
model,
test_loader,
weight=cfg.MODEL.WEIGHT,
tta=cfg.INFER.TTA)
import models, utils, datasets, predict
import logging, sys
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
scales = {
'Chicago': (0.78, 0.88, 1),
'Chicago RAC': (0.6, 0.8, 0.1),
'New Haven': (0.89, 1, 0.68),
'New Haven RAC': (1, 1, 1),
'Oakland': (0.84, 1, 0.51),
'Oakland RAC': (1, 0.94, 0.23),
'Richmond': (0.64, 1, 1),
'Richmond RAC': (1, 1, 1)
}
if __name__ == '__main__':
if len(sys.argv) == 1:
print "No model name was given! Run again using format: \n\t",
print "python test.py modelname"
else:
modelname = sys.argv[1]
pred = models.test_model(modelname)
categories = datasets.load_dataset('Categories')
n_pred = pred.shape[0]
pred = predict.apply_scales(pred, categories[-n_pred:], scales)
name = modelname + ".csv"
utils.create_submission(name, pred)
print "Saved submission with name %s" % (name)
def main(args, cfg):
logging.info(f"=========> {cfg.EXP} <=========")
# Declare variables
start_epoch = 0
best_metric = 100.
# Create model
model = get_model(cfg)
# Define Loss and Optimizer
train_criterion = nn.BCEWithLogitsLoss(weight=torch.FloatTensor(cfg.CONST.BCE_W))
valid_criterion = nn.BCEWithLogitsLoss(weight=torch.FloatTensor(cfg.CONST.BCE_W), reduction='none')
if cfg.TRAIN.NUM_CLASSES == 1:
train_criterion = nn.BCEWithLogitsLoss()
valid_criterion = nn.BCEWithLogitsLoss(reduction='none')
if args.valid:
valid_criterion = nn.BCELoss(weight=torch.FloatTensor(cfg.CONST.BCE_W), reduction='none')
optimizer = optim.AdamW(params=model.parameters(),
lr=cfg.OPT.BASE_LR,
weight_decay=cfg.OPT.WEIGHT_DECAY)
# CUDA & Mixed Precision
if cfg.SYSTEM.CUDA:
model = model.cuda()
train_criterion = train_criterion.cuda()
valid_criterion = valid_criterion.cuda()
if cfg.SYSTEM.FP16:
model, optimizer = amp.initialize(models=model, optimizers=optimizer,
opt_level=cfg.SYSTEM.OPT_L,
keep_batchnorm_fp32=(True if cfg.SYSTEM.OPT_L == "O2" else None))
# Load checkpoint
if args.load != "":
if os.path.isfile(args.load):
print(f"=> loading checkpoint {args.load}")
ckpt = torch.load(args.load, "cpu")
if cfg.TRAIN.MODEL == "octave-resnet50-hybrid":
model_dict = model.state_dict()
model_dict.update(ckpt.pop('state_dict'))
model.load_state_dict(model_dict)
else:
model.load_state_dict(ckpt.pop('state_dict'))
if not args.finetune:
print("resuming optimizer ...")
optimizer.load_state_dict(ckpt.pop('optimizer'))
start_epoch, best_metric = ckpt['epoch'], ckpt['best_metric']
logging.info(f"=> loaded checkpoint '{args.load}' (epoch {ckpt['epoch']}, best_metric: {ckpt['best_metric']})")
else:
logging.info(f"=> no checkpoint found at '{args.load}'")
if cfg.SYSTEM.MULTI_GPU:
model = nn.DataParallel(model)
# Load data
if cfg.TRAIN.CRNN:
DataSet = RSNAHemorrhageDS_RNN
else:
DataSet = RSNAHemorrhageDS
train_ds = DataSet(cfg, mode="train")
valid_ds = DataSet(cfg, mode="valid")
test_ds = DataSet(cfg, mode="test")
# Dataloader
valid_bs = 1 if cfg.TRAIN.CRNN else cfg.TRAIN.BATCH_SIZE
if cfg.DEBUG:
train_ds = Subset(train_ds, np.random.choice(np.arange(len(train_ds)), 256))
valid_ds = Subset(valid_ds, np.random.choice(np.arange(len(valid_ds)), 10))
test_ds = Subset(test_ds, np.random.choice(np.arange(len(test_ds)), 10))
train_loader = DataLoader(train_ds, cfg.TRAIN.BATCH_SIZE, pin_memory=False,
shuffle=True, drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
valid_loader = DataLoader(valid_ds, valid_bs, pin_memory=False,
shuffle=False, drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
test_loader = DataLoader(test_ds, valid_bs, pin_memory=False,
shuffle=False, drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
else:
if cfg.TRAIN.CRNN:
train_loader = DataLoader(train_ds, cfg.TRAIN.BATCH_SIZE, pin_memory=False,
shuffle=True,
drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
valid_loader = DataLoader(valid_ds, valid_bs, pin_memory=False, shuffle=False,
drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
test_loader = DataLoader(test_ds, valid_bs, pin_memory=False, shuffle=False,
drop_last=False, num_workers=cfg.SYSTEM.NUM_WORKERS)
else:
easy_img_ids = pd.read_csv(os.path.join(cfg.DIRS.DATA + "easy_train.csv"))["image"].values
easy_samples = train_ds.source[train_ds.source["image"].isin(easy_img_ids)]
hard_samples = train_ds.source[~train_ds.source["image"].isin(easy_img_ids)]
train_sampler = EasySampler(hard_samples.index, easy_samples.index, ratio=1.0)
train_loader = DataLoader(train_ds, cfg.TRAIN.BATCH_SIZE, pin_memory=False,
shuffle=False, sampler=train_sampler,
drop_last=False, num_workers=int(cfg.SYSTEM.NUM_WORKERS))
valid_loader = DataLoader(valid_ds, cfg.TRAIN.BATCH_SIZE, pin_memory=False, shuffle=False,
drop_last=False, num_workers=int(cfg.SYSTEM.NUM_WORKERS))
test_loader = DataLoader(test_ds, cfg.TRAIN.BATCH_SIZE, pin_memory=False, shuffle=False,
drop_last=False, num_workers=int(cfg.SYSTEM.NUM_WORKERS))
scheduler = LR_Scheduler("cos", cfg.OPT.BASE_LR, cfg.TRAIN.EPOCHS,\
iters_per_epoch=len(train_loader),
warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
if args.mode == "train":
train_loop(logging.info, cfg, model, \
train_loader, train_criterion, valid_loader, valid_criterion, \
optimizer, scheduler, start_epoch, best_metric)
elif args.mode == "valid":
valid_model(logging.info, cfg, model, valid_loader, valid_criterion, smooth_valid=cfg.INFER.SMOOTH, tta=cfg.INFER.TTA)
else:
test_model(logging.info, cfg, model, test_loader, smooth=cfg.INFER.SMOOTH, tta=cfg.INFER.TTA)
def main(args, cfg):
logging.info(f"=========> {cfg.EXP} <=========")
# Declare variables
start_epoch = 0
best_metric = 0.
# Create model
model = get_model(cfg)
# model = EfficientNet.from_pretrained(cfg.TRAIN.MODEL, num_classes=cfg.TRAIN.NUM_CLASSES)
if cfg.MODEL.SWA:
print("Get swa model.")
swa_model = get_model(cfg)
# Define Loss and Optimizer
if "bce" in cfg.TRAIN.LOSS:
train_criterion = nn.BCEWithLogitsLoss()
valid_criterion = nn.BCEWithLogitsLoss()
elif 'crossentropy' in cfg.TRAIN.LOSS:
train_criterion = nn.CrossEntropyLoss()
valid_criterion = nn.CrossEntropyLoss()
# #optimizer
optimizer = optim.AdamW(params=model.parameters(),
lr=cfg.OPT.BASE_LR,
weight_decay=cfg.OPT.WEIGHT_DECAY)
if cfg.OPT.CLR:
optimizer = optim.SGD(params=model.parameters(),
lr=cfg.OPT.BASE_LR,
momentum=0.9)
# CUDA & Mixed Precision
if cfg.SYSTEM.CUDA:
if cfg.MODEL.SWA:
print("Swa model to cuda, bitch")
swa_model.cuda()
model = model.cuda()
train_criterion = train_criterion.cuda()
valid_criterion = valid_criterion.cuda()
if cfg.SYSTEM.FP16:
if cfg.MODEL.SWA:
print("FP16 swa")
[model, swa_model], optimizer = amp.initialize(
models=[model, swa_model],
optimizers=optimizer,
opt_level=cfg.SYSTEM.OPT_L,
keep_batchnorm_fp32=(True
if cfg.SYSTEM.OPT_L == "O2" else None))
else:
model, optimizer = amp.initialize(
models=model,
optimizers=optimizer,
opt_level=cfg.SYSTEM.OPT_L,
keep_batchnorm_fp32=(True
if cfg.SYSTEM.OPT_L == "O2" else None))
# Load checkpoint
if args.load != "":
if os.path.isfile(args.load):
print(f"=> loading checkpoint {args.load}")
ckpt = torch.load(args.load, "cpu")
try:
print("Loaded SRnet pretrained.")
# model.load_state_dict(ckpt['model_state_dict'])
state_dict = ckpt['model_state_dict']
#pop 1st layer to train different channel
if cfg.DATA.INP_CHANNEL != 1:
state_dict.pop("layer1.weight")
#pop fc out of weight to train multiple classes
state_dict.pop('fc.weight')
state_dict.pop('fc.bias')
model.load_state_dict(state_dict, strict=False)
except:
model.load_state_dict(ckpt.pop('state_dict'))
if not args.finetune:
print("resuming optimizer ...")
optimizer.load_state_dict(ckpt.pop('optimizer'))
start_epoch, best_metric = ckpt['epoch'], ckpt[
'best_metric']
logging.info(
f"=> loaded checkpoint '{args.load}' (epoch {ckpt['epoch']}, best_metric: {ckpt['best_metric']})"
)
if cfg.MODEL.SWA:
ckpt = torch.load(args.load, "cpu")
swa_model.load_state_dict(ckpt.pop('state_dict'))
else:
logging.info(f"=> no checkpoint found at '{args.load}'")
if cfg.SYSTEM.MULTI_GPU:
model = nn.DataParallel(model)
# Load data
train_loader = get_dataset(cfg, 'train')
valid_loader = get_dataset(cfg, 'valid')
test_loader = get_dataset(cfg, 'test')
if cfg.DEBUG:
train_loader = get_debug_dataset(cfg, 'train')
valid_loader = get_debug_dataset(cfg, 'valid')
#lr finder
# lr_finder = LRFinder(model, optimizer, train_criterion, device='cuda')
# lr_finder.range_test(train_loader, end_lr=10, num_iter=100, step_mode='exp', accumulation_steps=cfg.OPT.GD_STEPS)
# lr_finder.plot()
# lr_finder.reset()
if cfg.OPT.CLR:
print("Use Cyclical learning rate scheduler, bitch.")
# scheduler = WarmupCyclicalLR("cos", cfg.OPT.BASE_LR, cfg.TRAIN.EPOCHS,
# iters_per_epoch=len(train_loader),
# warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
iter_per_ep = len(train_loader)
scheduler = torch.optim.lr_scheduler.CyclicLR(
optimizer, base_lr=0.03, max_lr=0.1, step_size_up=iter_per_ep // 2)
elif cfg.OPT.WARMUP_CLR:
print("Use warmup cyclical LR, bitch")
scheduler = WarmupCyclicalLR("cos",
cfg.OPT.BASE_LR,
cfg.TRAIN.EPOCHS,
iters_per_epoch=len(train_loader),
warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
else:
print("Use cosine learning rate scheduler, bitch")
scheduler = LR_Scheduler("cos", cfg.OPT.BASE_LR, cfg.TRAIN.EPOCHS,\
iters_per_epoch=len(train_loader),
warmup_epochs=cfg.OPT.WARMUP_EPOCHS)
if cfg.DATA.PSEUDO:
print("Training with pseudo label.")
if args.mode == "train" and cfg.MODEL.SWA == False:
train_loop(logging.info, cfg, model, \
train_loader, valid_loader, train_criterion, valid_criterion,\
optimizer, scheduler, start_epoch, best_metric)
elif args.mode == "train" and cfg.MODEL.SWA == True:
time_all = time.time()
print('SWA training')
for epoch in range(start_epoch, cfg.TRAIN.EPOCHS):
time_ep = time.time()
swa_train_loop(logging.info, cfg, model, swa_model, \
train_loader, valid_loader, train_criterion, valid_criterion,\
optimizer, scheduler, epoch, best_metric)
time_ep = time.time() - time_ep
print(f"Time epoch: {round(time_ep,4)} seconds")
print(f"Finished training in: {round(time_all, 4)} mins")
elif args.mode == "valid":
valid_model(logging.info,
cfg,
model,
valid_criterion,
valid_loader,
tta=cfg.INFER.TTA)
else:
test_model(logging.info, cfg, model, test_loader, tta=cfg.INFER.TTA)
