def visualise_annotations(source_dirs, n=2, target_colour=None):
# Extract filenames for images/masks
dirs = [
os.path.join(dir, f) for dir in source_dirs for f in os.listdir(dir)
]
files = [
os.path.join(f, 'images', ''.join((os.path.split(f)[-1], '.png')))
for f in dirs
]
maskfiles = [[
os.path.join(f, 'masks', m)
for m in os.listdir(os.path.join(f, 'masks'))
] if os.path.exists(os.path.join(f, 'masks')) else None for f in dirs]
# Reduce to a target colour if requested
if target_colour is not None:
from dataset import get_ids
colour_id, _ = get_ids(files)
valid_idx = np.argwhere(du.ismember(colour_id,
np.array(target_colour))).reshape(
-1, )
files = [files[idx] for idx in valid_idx]
maskfiles = [maskfiles[idx] for idx in valid_idx]
img_list = []
counter = 0
for f, m in zip(files, maskfiles):
img = load_img(f)
masks = np.stack([imageio.imread(path) for path in m], axis=-1)
labels = du.maskrcnn_mask_to_labels(masks)
counter += 1
if counter > n:
# Display
plot_multiple_images(img_list, nrows=n, ncols=2)
# Reset
counter = 0
img_list = []
else:
img_list.extend([img, image_with_labels(img, labels)])
return
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('mode', choices=['train', 'validate', 'predict_valid', 'predict_test'])
arg('run_root')
arg('--model', default='se_resnet50')
arg('--pretrained', type=int, default=1)
arg('--batch-size', type=int, default=64)
arg('--step', type=int, default=1)
arg('--workers', type=int, default=2 if ON_KAGGLE else 4)
arg('--lr', type=float, default=1e-4)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=100)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=4)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
arg('--fold', type=int, default=0)
arg('--multi', type=int, default=0)
arg('--half', type=int, default=0)
args = parser.parse_args()
run_root = Path(args.run_root)
folds = pd.read_csv('../folds.csv')
train_root = DATA_ROOT / ('train_sample' if args.use_sample else 'train')
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
train_fold = folds[folds['fold'] != args.fold]
valid_fold = folds[folds['fold'] == args.fold]
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
def make_loader(df: pd.DataFrame, image_transform) -> DataLoader:
return DataLoader(
TrainDataset(train_root, df, image_transform, debug=args.debug),
shuffle=True,
batch_size=args.batch_size,
num_workers=args.workers,
)
criterion = nn.BCEWithLogitsLoss(reduction='none')
# criterion = FocalLoss()
model = getattr(models, args.model)()
feature_dim = model.last_linear.in_features
class AvgPool(nn.Module):
def forward(self, x):
# print (x.size())
return F.avg_pool2d(x, x.shape[2:])
model.avg_pool = AvgPool()
model.avgpool = AvgPool()
model.last_linear = nn.Linear(feature_dim, N_CLASSES)
use_cuda = cuda.is_available()
fresh_params = list(model.last_linear.parameters())
all_params = list(model.parameters())
if use_cuda:
if args.multi:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.cuda()
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
(run_root / 'params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
writer = SummaryWriter(str(run_root/'tbx'))
train_loader = make_loader(train_fold, train_transform)
valid_loader = make_loader(valid_fold, test_transform)
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr),
use_cuda=use_cuda,
)
if args.pretrained:
if train(params=fresh_params, writer=writer, n_epochs=1, **train_kwargs):
train(params=all_params, writer=writer, **train_kwargs)
else:
train(params=all_params, writer=writer, **train_kwargs)
writer.export_scalars_to_json(str(run_root/'tbx/all_scalars.json'))
writer.close()
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, test_transform)
load_model(model, run_root / 'model.pt')
validation(model, criterion, tqdm.tqdm(valid_loader, desc='Validation'),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
load_model(model, run_root / 'best-model.pt')
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
predict(model, df=valid_fold, root=train_root,
out_path=run_root / 'val.h5',
**predict_kwargs)
elif args.mode == 'predict_test':
test_root = DATA_ROOT / (
'test_sample' if args.use_sample else 'test')
ss = pd.read_csv(DATA_ROOT / 'sample_submission.csv')
if args.use_sample:
ss = ss[ss['id'].isin(set(get_ids(test_root)))]
if args.limit:
ss = ss[:args.limit]
predict(model, df=ss, root=test_root,
out_path=run_root / 'test.h5',
**predict_kwargs)
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--mode',
choices=['train', 'validate', 'predict_valid', 'predict_test'],
default='train')
arg('--run_root', default='result')
arg('--model', default='se_resnext101')
arg('--pretrained', type=str,
default='imagenet') #resnet 1, resnext imagenet
arg('--batch-size', type=int, default=80)
arg('--step', type=str, default=1)
arg('--workers', type=int, default=16)
arg('--lr', type=float, default=1e-5)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=25)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=1)
arg('--use-sample',
action='store_true',
help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
arg('--fold', type=int, default=2)
arg('--ckpt', type=str, default='')
args = parser.parse_args()
run_root = Path(args.run_root)
folds = pd.read_csv('folds_skf.csv')
train_root = DATA_ROOT + '/' + ('train_sample'
if args.use_sample else 'train')
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
train_fold = folds[folds['fold'] != args.fold]
valid_fold = folds[folds['fold'] == args.fold]
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
def make_loader(df: pd.DataFrame,
image_transform,
name='train') -> DataLoader:
return DataLoader(
TrainDataset(train_root,
df,
image_transform,
debug=args.debug,
name='train'),
shuffle=True,
batch_size=args.batch_size,
num_workers=args.workers,
)
criterion = nn.BCEWithLogitsLoss(reduction='none')
#if args.model != 'se_resnext50' and args.model != 'se_resnext101':
if 'se' not in args.model:
model = getattr(models, args.model)(num_classes=N_CLASSES,
pretrained=args.pretrained)
else:
model = getattr(models, args.model)(num_classes=N_CLASSES,
pretrained='imagenet')
use_cuda = cuda.is_available()
#if args.model != 'se_resnext50' and args.model != 'se_resnext101':
if 'se' not in args.model:
fresh_params = list(model.fresh_params())
all_params = list(model.parameters())
if torch.cuda.device_count() > 1:
model = torch.nn.DataParallel(model)
if use_cuda:
model = model.cuda()
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
Path(str(run_root) + '/params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
train_loader = make_loader(train_fold, train_transform, name='train')
valid_loader = make_loader(valid_fold, test_transform, name='valid')
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr),
#init_optimizer=lambda params, lr: SGD(params, lr, momentum = 0.9),
use_cuda=use_cuda,
)
#if args.pretrained and args.model != 'se_resnext50' and args.model != 'se_resnext101':
if 'se' not in args.model and args.pretrained:
if train(params=fresh_params, n_epochs=1, **train_kwargs):
train(params=all_params, **train_kwargs)
else:
train(params=all_params, **train_kwargs)
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, test_transform, name='valid')
load_model(model, Path(str(run_root) + '/' + 'model.pt'))
validation(model,
criterion,
tqdm.tqdm(valid_loader, desc='Validation'),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
load_model(model, Path(str(run_root) + '/tmp_ckpt/' + args.ckpt))
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
predict(model,
df=valid_fold,
root=train_root,
out_path=Path(str(run_root) + '/' + 'val.h5'),
**predict_kwargs)
elif args.mode == 'predict_test':
test_root = DATA_ROOT + '/' + ('test_sample'
if args.use_sample else 'test')
ss = pd.read_csv(DATA_ROOT + '/' + 'sample_submission.csv')
if args.use_sample:
ss = ss[ss['id'].isin(set(get_ids(test_root)))]
if args.limit:
ss = ss[:args.limit]
predict(model,
df=ss,
root=test_root,
out_path=Path(
str(run_root) + '/fold' + str(args.fold) + '_test.h5'),
**predict_kwargs)
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--mode', choices=['train', 'validate', 'predict_valid', 'predict_test'], default='train')
arg('--run_root', default='result/furniture_toy')
arg('--fold', type=int, default=0)
arg('--model', default='resnet50V3')
arg('--ckpt', type=str, default='model_loss_best.pt')
arg('--pretrained', type=str, default='imagenet')#resnet 1, resnext imagenet
arg('--batch-size', type=int, default=32)
arg('--step', type=str, default=8)
arg('--workers', type=int, default=16)
arg('--lr', type=float, default=3e-5)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=60)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=1)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
arg('--imgsize',type=int, default = 256)
args = parser.parse_args()
run_root = Path(args.run_root)
folds = pd.read_csv('train_val_test_furniture.csv')
# train_root = DATA_ROOT + '/' + ('train_sample' if args.use_sample else 'train')
train_root = DATA_ROOT
# valid_root = DATA_ROOT + '/' + 'validate'
valid_root = DATA_ROOT
#Only images in train_sample are used folds = folds[bool vec]
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
train_fold = folds[folds['fold'] == 0]
valid_fold = folds[folds['fold'] == 1]
#limit the size of train/valid data
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
#DataLoader
def make_loader(df: pd.DataFrame, root, image_transform, name='train') -> DataLoader:
return DataLoader(
TrainDataset(root, df, debug=args.debug, name=name, imgsize = args.imgsize),
shuffle=True,
batch_size=args.batch_size,
num_workers=args.workers,
)
criterion = nn.BCEWithLogitsLoss(reduction='none')
# criterion = nn.CrossEnropyLoss(reduction='none)
if 'se' not in args.model and 'ception' not in args.model and 'dpn' not in args.model:
model = getattr(models, args.model)(
num_classes=N_CLASSES, pretrained=args.pretrained)
else:
model = getattr(models, args.model)(
num_classes=N_CLASSES, pretrained='imagenet')
use_cuda = cuda.is_available()
if 'se' not in args.model and 'ception' not in args.model and 'dpn' not in args.model:
fresh_params = list(model.fresh_params())
all_params = list(model.parameters())
model = torch.nn.DataParallel(model)
if use_cuda:
model = model.cuda()
#print(model)
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
Path(str(run_root) + '/params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
train_loader = make_loader(train_fold, train_root, train_transform, name='train')
valid_loader = make_loader(valid_fold, valid_root, test_transform, name='valid')
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr, betas=(0.9,0.999), eps=1e-08, weight_decay = 2e-4),
use_cuda=use_cuda,
)
#if args.pretrained and args.model != 'se_resnext50' and args.model != 'se_resnext101':
if 'se' not in args.model and 'ception' not in args.model and args.pretrained and 'dpn' not in args.model:
if train(params=fresh_params, n_epochs=1, **train_kwargs):
train(params=all_params, **train_kwargs)
else:
train(params=all_params, **train_kwargs)
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, valid_root ,image_transform=test_transform, name='valid')
load_model(model, Path(str(run_root) + '/' + args.ckpt))
validation(model, criterion, tqdm.tqdm(valid_loader, desc='Validation'),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
load_model(model, Path(str(run_root) + '/' + args.ckpt))
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
predict(model, df=valid_fold, root=train_root,
out_path=Path(str(run_root) + '/' + 'val.h5'),
**predict_kwargs)
elif args.mode == 'predict_test':
load_model(model, Path(str(run_root) + '/' + args.ckpt))
print(args.ckpt)
test_root = DATA_ROOT + '/' + (
'test_sample' if args.use_sample else 'test2')
test_df = pd.read_csv('train_val_test.csv')
test_df = test_df[test_df['fold'] == 2]
tta_code_list = []
tta_code_list.append([0, 0])
tta_code_list.append([0, 1])
tta_code_list.append([0, 2])
tta_code_list.append([0, 3])
tta_code_list.append([0, 4])
tta_code_list.append([1, 0])
tta_code_list.append([1, 1])
tta_code_list.append([1, 2])
tta_code_list.append([1, 3])
tta_code_list.append([1, 4])
tta_code_list.append([0, 5])
tta_code_list.append([1, 5])
save_dir = str(run_root) + '/12tta'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for tta_code in tta_code_list:
print(tta_code)
predict(model,
df=test_df,
root=test_root,
out_path=Path(str(run_root) + '/12tta/fold' + str(args.fold)+'_'+str(tta_code[0]) + str(tta_code[1]) + '_test.h5'),
batch_size = args.batch_size,
tta_code=tta_code,
workers=8,
use_cuda=True)
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--mode', choices=['train', 'validate', 'predict_valid', 'predict_test'], default='train')
arg('--run_root', default='result/eff_b3_fold9')
arg('--fold', type=int, default=9)
arg('--model', default='eff_b3')
arg('--ckpt', type=str, default='')
arg('--pretrained', type=str, default='imagenet')#resnet 1, resnext imagenet
arg('--batch-size', type=int, default=40)
arg('--step', type=str, default=1)
arg('--workers', type=int, default=8)
arg('--lr', type=float, default=3e-4)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=25)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=1)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
args = parser.parse_args()
run_root = Path(args.run_root)
folds = pd.read_csv('folds_skf.csv')
train_root = DATA_ROOT + '/' + ('train_sample' if args.use_sample else 'train')
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
train_fold = folds[folds['fold'] != args.fold]
valid_fold = folds[folds['fold'] == args.fold]
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
def make_loader(df: pd.DataFrame, image_transform, name='train') -> DataLoader:
return DataLoader(
TrainDataset(train_root, df, debug=args.debug, name=name),
shuffle=True,
batch_size=args.batch_size,
num_workers=16,
)
criterion = nn.BCEWithLogitsLoss(reduction='none')
if 'se' not in args.model and 'ception' not in args.model and 'dpn' not in args.model and 'eff' not in args.model:
model = getattr(models, args.model)(
num_classes=N_CLASSES, pretrained=args.pretrained)
elif 'eff' in args.model:
model = eff_b3 = EfficientNet.from_name('efficientnet-b3')
model.load_state_dict(torch.load('efficientnet-b3-c8376fa2.pth'))
model._fc = nn.Linear(1536, N_CLASSES)
else:
model = getattr(models, args.model)(
num_classes=N_CLASSES, pretrained='imagenet')
use_cuda = cuda.is_available()
if 'se' not in args.model and 'ception' not in args.model and 'dpn' not in args.model and 'eff' not in args.model:
fresh_params = list(model.fresh_params())
all_params = list(model.parameters())
model = torch.nn.DataParallel(model)
if use_cuda:
model = model.cuda()
print(model)
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
Path(str(run_root) + '/params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
train_loader = make_loader(train_fold, train_transform, name='train')
valid_loader = make_loader(valid_fold, test_transform, name='valid')
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr),
use_cuda=use_cuda,
)
#if args.pretrained and args.model != 'se_resnext50' and args.model != 'se_resnext101':
if 'se' not in args.model and 'ception' not in args.model and args.pretrained and 'dpn' not in args.model and 'eff' not in args.model:
if train(params=fresh_params, n_epochs=1, **train_kwargs):
train(params=all_params, **train_kwargs)
else:
train(params=all_params, **train_kwargs)
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, test_transform, name='valid')
load_model(model, Path(str(run_root) + '/' + args.ckpt))
validation(model, criterion, tqdm.tqdm(valid_loader, desc='Validation'),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
load_model(model, Path(str(run_root) + '/' + args.ckpt))
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
predict(model, df=valid_fold, root=train_root,
out_path=Path(str(run_root) + '/' + 'val.h5'),
**predict_kwargs)
elif args.mode == 'predict_test':
# ckpt_list = [r'se101_skf_fold0_0.6140.pt',
# r'se101_skf_fold1_0.6137.pt',
# r'se101_skf_fold2_0.6107.pt',
# r'se101_skf_fold3_0.6175.pt',
# r'se101_skf_fold4_0.6149.pt',
# r'se101_skf_fold5_0.6151.pt',
# r'se101_skf_fold6_0.6179.pt',
# r'se101_skf_fold7_0.6199.pt',
# r'se101_skf_fold8_0.6159.pt',
# r'se101_skf_fold9_0.6116.pt',
# ]
# for tmp in ckpt_list:
# args.ckpt = tmp
load_model(model, Path(str(run_root) + '/' + args.ckpt))
print(args.ckpt)
test_root = DATA_ROOT + '/' + (
'test_sample' if args.use_sample else 'test')
ss = pd.read_csv(DATA_ROOT + '/' + 'sample_submission.csv')
if args.use_sample:
ss = ss[ss['id'].isin(set(get_ids(test_root)))]
if args.limit:
ss = ss[:args.limit]
tta_code_list = []
tta_code_list.append([0, 0])
tta_code_list.append([0, 1])
tta_code_list.append([0, 2])
tta_code_list.append([0, 3])
tta_code_list.append([0, 4])
tta_code_list.append([1, 0])
tta_code_list.append([1, 1])
tta_code_list.append([1, 2])
tta_code_list.append([1, 3])
tta_code_list.append([1, 4])
tta_code_list.append([0, 5])
tta_code_list.append([1, 5])
save_dir = str(run_root) + '/112tta'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for tta_code in tta_code_list:
print(tta_code)
predict(model,
df=ss,
root=test_root,
out_path=Path(str(run_root) + '/112tta/fold' + str(args.fold)+'_'+str(tta_code[0]) + str(tta_code[1]) + '_test.h5'),
batch_size = args.batch_size,
tta_code=tta_code,
workers=16,
use_cuda=True)
def main(*args):
#def main():
# print("do main")
parser = argparse.ArgumentParser()
arg = parser.add_argument
# TODO : "--modeにGradCAMを追加"
arg('--mode', choices=['train', 'validate', 'predict_valid', 'predict_test'])
arg('--run_root')
arg('--model', default='resnet50')
arg('--loss',default="focalloss")
arg('--pretrained', type=int, default=1)
arg('--batch-size', type=int, default=64)
arg('--step', type=int, default=1)
arg('--workers', type=int, default=4 if ON_KAGGLE else 4)
arg('--lr', type=float, default=1e-4)
arg('--patience', type=int, default=4)
arg('--clean', action='store_true')
arg('--n-epochs', type=int, default=100)
arg('--epoch-size', type=int)
arg('--tta', type=int, default=4)
arg('--use-sample', action='store_true', help='use a sample of the dataset')
arg('--debug', action='store_true')
arg('--limit', type=int)
arg('--fold', type=int, default=0)
arg('--regression',type=int,default=0)
arg('--finetuning',type=int,default=1)
# TODO : classificationかregressionかをオプションで追加できるようにする。
# from IPython.core.debugger import Pdb; Pdb().set_trace()
if is_env_notebook():
args = parser.parse_args(args=args[0])
else:
args = parser.parse_args()
# from IPython.core.debugger import Pdb; Pdb().set_trace()
run_root = Path(args.run_root)
folds = pd.read_csv('folds.csv')
train_root = DATA_ROOT / ('train_sample' if args.use_sample else 'train_images')
if args.use_sample:
folds = folds[folds['Id'].isin(set(get_ids(train_root)))]
# Pdb().set_trace()
# -1 はleakデータ
train_fold = folds[folds['fold'] != args.fold]
leak_fold = folds[folds['fold'] == -1]
train_fold = pd.concat([train_fold,leak_fold])
valid_fold = folds[folds['fold'] == args.fold]
if args.limit:
train_fold = train_fold[:args.limit]
valid_fold = valid_fold[:args.limit]
def make_loader(df: pd.DataFrame, image_transform,regression=args.regression,shuffle=False,balanced=True) -> DataLoader:
return DataLoader(
TrainDataset(train_root, df, image_transform, debug=args.debug,regression=regression,balanced=balanced),
shuffle=shuffle,
batch_size=args.batch_size,
num_workers=args.workers,
)
## TODO : regressionようにモデルを書き換え
if args.regression:
criterion = nn.MSELoss()
# TODO : 回帰モデルへ変更
model = getattr(models, args.model)(
num_classes=1, pretrained=args.pretrained)
else:
# 分類モデル
criterion = FocalLoss()#nn.BCEWithLogitsLoss(reduction='none')
model = getattr(models, args.model)(
num_classes=N_CLASSES, pretrained=args.pretrained)
# Pdb().set_trace()
use_cuda = cuda.is_available()
fresh_params = list(model.fresh_params())
all_params = list(model.parameters())
if use_cuda:
model = model.cuda()
if args.mode == 'train':
if run_root.exists() and args.clean:
shutil.rmtree(run_root)
run_root.mkdir(exist_ok=True, parents=True)
(run_root / 'params.json').write_text(
json.dumps(vars(args), indent=4, sort_keys=True))
train_loader = make_loader(train_fold, train_transform,regression=args.regression,balanced=True)
valid_loader = make_loader(valid_fold, test_transform,regression=args.regression,balanced=False)
print(f'{len(train_loader.dataset):,} items in train, '
f'{len(valid_loader.dataset):,} in valid')
train_kwargs = dict(
args=args,
model=model,
criterion=criterion,
train_loader=train_loader,
valid_loader=valid_loader,
patience=args.patience,
init_optimizer=lambda params, lr: Adam(params, lr),
use_cuda=use_cuda,
)
# from IPython.core.debugger import Pdb; Pdb().set_trace()
if args.pretrained:
if train(params=fresh_params, n_epochs=1, **train_kwargs):
train(params=all_params, **train_kwargs)
else:
train(params=all_params, **train_kwargs)
# fine-tunig after balanced learning
if args.finetuning:
print("Start Fine-tuning")
TUNING_EPOCH = 5
train_loader = make_loader(train_fold, train_transform,regression=args.regression,balanced=False)
# 学習率を小さくする
args.lr = args.lr / 5
train_kwargs["train_loader"] = train_loader
train(params=all_params,n_epochs=args.n_epochs+TUNING_EPOCH,**train_kwargs,finetuning=args.finetuning)
elif args.mode == 'validate':
valid_loader = make_loader(valid_fold, test_transform)
load_model(model, run_root / 'model.pt')
validation(model, criterion, tqdm.tqdm(valid_loader, desc='Validation',valid_fold=valid_fold),
use_cuda=use_cuda)
elif args.mode.startswith('predict'):
print("load model predict")
load_model(model, run_root / 'best-model.pt')
predict_kwargs = dict(
batch_size=args.batch_size,
tta=args.tta,
use_cuda=use_cuda,
workers=args.workers,
)
if args.mode == 'predict_valid':
#predict(model, df=valid_fold, root=train_root,
# out_path=run_root / 'val.h5',
# **predict_kwargs)
valid_loader = make_loader(valid_fold, test_transform,shuffle=False,balanced=False)
#model: nn.Module, criterion, valid_loader, use_cuda,valid_predict:bool=False
# TODO : valid foldに予測結果をくっ付ける操作を追加
validation(model,criterion,valid_loader,use_cuda,valid_fold=valid_fold,valid_predict=True,save_path=run_root)
elif args.mode == 'predict_test':
test_root = DATA_ROOT / (
'test_sample' if args.use_sample else 'test_images')
ss = pd.read_csv(DATA_ROOT / 'sample_submission.csv')
if args.use_sample:
ss = ss[ss['id'].isin(set(get_ids(test_root)))]
if args.limit:
ss = ss[:args.limit]
predict(model, df=ss, root=test_root,
out_path=run_root / 'test.h5',
**predict_kwargs)