def main():
import warnings
warnings.filterwarnings("ignore")
print('start training.')
ymls = ['configs/infer.yml']
for yml in ymls:
config = utils.config.load(yml)
os.environ["CUDA_VISIBLE_DEVICES"] = '0' #,1' # config.GPU
prepare_train_directories(config)
pprint.pprint(config, indent=2)
utils.config.save_config(yml, config.TRAIN_DIR + config.RECIPE)
run(config)
def main():
import warnings
warnings.filterwarnings("ignore")
print('start training.')
seed_everything()
yml = 'configs/base.yml'
config = utils.config.load(yml)
prepare_train_directories(config)
pprint.pprint(config, indent=2)
shutil.copy(yml, os.path.join(config.TRAIN_DIR, 'config.yml'))
run(config)
print('success!')
def main():
import warnings
warnings.filterwarnings("ignore")
print('start training.')
seed_everything()
yml = 'configs/' + sys.argv[1]
config = utils.config.load(yml)
prepare_train_directories(config)
pprint.pprint(config, indent=2)
utils.config.save_config(yml, config.TRAIN_DIR)
run(config)
print('success!')
def main():
import warnings
warnings.filterwarnings("ignore")
print('start training.')
seed_everything()
ymls = ['configs/fastscnn_mv3_sj_add_data_1024.yml']
for yml in ymls:
config = utils.config.load(yml)
os.environ["CUDA_VISIBLE_DEVICES"] = '0,1' #config.GPU
prepare_train_directories(config)
pprint.pprint(config, indent=2)
utils.config.save_config(yml, config.TRAIN_DIR + config.RECIPE)
run(config)
print('success!')
def main():
seed_everything()
# yml = 'configs/base.yml'
# config = utils.config.load(yml)
# pprint.pprint(config, indent=2)
model = get_model(config).cuda()
bind_model(model)
args = get_args()
if args.pause: ## test mode일 때
print('Inferring Start...')
nsml.paused(scope=locals())
if args.mode == 'train': ### training mode일 때
print('Training Start...')
# no bias decay
if config.OPTIMIZER.NO_BIAS_DECAY:
group_decay, group_no_decay = group_weight(model)
params = [{'params': group_decay},
{'params': group_no_decay, 'weight_decay': 0.0}]
else:
params = model.parameters()
optimizer = get_optimizer(config, params)
optimizer.param_groups[0]['initial_lr'] = config.OPTIMIZER.LR
if config.OPTIMIZER.NO_BIAS_DECAY:
optimizer.param_groups[1]['initial_lr'] = config.OPTIMIZER.LR
###############################################################################################
if IS_LOCAL:
prepare_train_directories(config)
utils.config.save_config(yml, config.LOCAL_TRAIN_DIR)
checkpoint = utils.checkpoint.get_initial_checkpoint(config)
if checkpoint is not None:
last_epoch, score, loss = utils.checkpoint.load_checkpoint(config, model, checkpoint)
else:
print('[*] no checkpoint found')
last_epoch, score, loss = -1, -1, float('inf')
print('last epoch:{} score:{:.4f} loss:{:.4f}'.format(last_epoch, score, loss))
else:
last_epoch = -1
###############################################################################################
scheduler = get_scheduler(config, optimizer, last_epoch=last_epoch)
###############################################################################################
if IS_LOCAL:
if config.SCHEDULER.NAME == 'multi_step':
if config.SCHEDULER.WARMUP:
scheduler_dict = scheduler.state_dict()['after_scheduler'].state_dict()
else:
scheduler_dict = scheduler.state_dict()
milestones = scheduler_dict['milestones']
step_count = len([i for i in milestones if i < last_epoch])
optimizer.param_groups[0]['lr'] *= scheduler_dict['gamma'] ** step_count
if config.OPTIMIZER.NO_BIAS_DECAY:
optimizer.param_groups[1]['initial_lr'] *= scheduler_dict['gamma'] ** step_count
if last_epoch != -1:
scheduler.step()
###############################################################################################
# for dirname, _, filenames in os.walk(DATASET_PATH):
# for filename in filenames:
# print(os.path.join(dirname, filename))
# if preprocessing possible
preprocess_type = config.DATA.PREPROCESS
cv2_size = (config.DATA.IMG_W, config.DATA.IMG_H)
if not IS_LOCAL:
preprocess(os.path.join(DATASET_PATH, 'train', 'train_data', 'NOR'), os.path.join(preprocess_type, 'NOR'), preprocess_type, cv2_size)
preprocess(os.path.join(DATASET_PATH, 'train', 'train_data', 'AMD'), os.path.join(preprocess_type, 'AMD'), preprocess_type, cv2_size)
preprocess(os.path.join(DATASET_PATH, 'train', 'train_data', 'RVO'), os.path.join(preprocess_type, 'RVO'), preprocess_type, cv2_size)
preprocess(os.path.join(DATASET_PATH, 'train', 'train_data', 'DMR'), os.path.join(preprocess_type, 'DMR'), preprocess_type, cv2_size)
data_dir = preprocess_type
# data_dir = os.path.join(DATASET_PATH, 'train/train_data')
else: # IS_LOCAL
data_dir = os.path.join(DATASET_PATH, preprocess_type)
# eda
# train_std(data_dir, preprocess_type, cv2_size)
fold_df = split_cv(data_dir, n_splits=config.NUM_FOLDS)
val_fold_idx = config.IDX_FOLD
###############################################################################################
train_loader = get_dataloader(config, data_dir, fold_df, val_fold_idx, 'train', transform=Albu())
val_loader = get_dataloader(config, data_dir, fold_df, val_fold_idx, 'val')
postfix = dict()
num_epochs = config.TRAIN.NUM_EPOCHS
val_acc_list = []
for epoch in range(last_epoch+1, num_epochs):
if epoch >= config.LOSS.FINETUNE_EPOCH:
criterion = get_loss(config.LOSS.FINETUNE_LOSS)
else:
criterion = get_loss(config.LOSS.NAME)
train_values = train_single_epoch(config, model, train_loader, criterion, optimizer, scheduler, epoch)
val_values = evaluate_single_epoch(config, model, val_loader, criterion, epoch)
val_acc_list.append((epoch, val_values[2]))
if config.SCHEDULER.NAME != 'one_cyle_lr':
scheduler.step()
if IS_LOCAL:
utils.checkpoint.save_checkpoint(config, model, epoch, val_values[1], val_values[0])
else:
postfix['train_loss'] = train_values[0]
postfix['train_res'] = train_values[1]
postfix['train_acc'] = train_values[2]
postfix['train_sens'] = train_values[3]
postfix['train_spec'] = train_values[4]
postfix['val_loss'] = val_values[0]
postfix['val_res'] = val_values[1]
postfix['val_acc'] = val_values[2]
postfix['val_sens'] = val_values[3]
postfix['val_spec'] = val_values[4]
nsml.report(**postfix, summary=True, step=epoch)
val_res = '%.10f' % val_values[1]
val_res = val_res.replace(".", "")
val_res = val_res[:4] + '.' + val_res[4:8] + '.' + val_res[8:]
save_name = 'epoch_%02d_score%s_loss%.4f.pth' % (epoch, val_res, val_values[0])
# nsml.save(save_name)
nsml.save(epoch)
for e, val_acc in val_acc_list:
print('%02d %s' % (e, val_acc))
def main():
import warnings
warnings.filterwarnings("ignore")
print('start training.')
seed_everything()
ymls = ['configs/base.yml']
yml = ymls[0]
config = utils.config.load(yml)
if not isLocal:
## server so single.. not paraller
print("!!!!!!!!!!!No Parallel!!!!!!!!!!")
config.PARALLEL = False
# exp. input_size
input_size_list = config.EXPERIMENT.INPUT_SIZE_LIST
batch_size_list = config.EXPERIMENT.BATCH_SIZE_LIST
model_list = config.EXPERIMENT.MODEL_LIST
original_recipe = config.RECIPE
for input_size in input_size_list:
config.DATA.IMG_H = input_size
config.DATA.IMG_W = input_size
for batch_size in batch_size_list:
config.TRAIN.BATCH_SIZE = batch_size
config.EVAL.BATCH_SIZE = batch_size * 2
for selected_model_name in model_list:
# config.TRAIN.BATCH_SIZE = batch_size
config.MODEL.NAME = selected_model_name
if selected_model_name == "EfficientNetB7":
config.TRAIN.BATCH_SIZE = 4
config.EVAL.BATCH_SIZE = config.TRAIN.BATCH_SIZE * 2
config.MODEL.WEIGHT = 'noisy-student'
if config.DEBUG:
config.RECIPE = "debug_" + original_recipe
else:
config.RECIPE = original_recipe
## recipe set batch_input_
config.RECIPE = "_".join(
(config.RECIPE, config.MODEL.NAME,
'B' + str(config.TRAIN.BATCH_SIZE), 'S' +
str(config.DATA.IMG_W) + 'x' + str(config.DATA.IMG_H),
'LR' + str(config.OPTIMIZER.LR)))
##delete log_settings
if config.DEBUG or config.DELETE_LOGPATH:
path = os.path.join(config.TRAIN_DIR, config.RECIPE)
if os.path.exists(path):
shutil.rmtree(path)
print("Delete path: ", path)
##delete all debug path
train_path_list = os.listdir(config.TRAIN_DIR)
for path_recipe in train_path_list:
if 'debug' in path_recipe:
path = os.path.join(config.TRAIN_DIR, path_recipe)
if os.path.exists(path):
shutil.rmtree(path)
print("Delete debug_path: " + path)
prepare_train_directories(config)
pprint.pprint(config, indent=2)
utils.config.save_config(yml, config.TRAIN_DIR + config.RECIPE)
run(config)
print('success!')