def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('root', help='checkpoint root')
arg('--batch-size', type=int, default=32)
arg('--patch-size', type=int, default=256)
arg('--n-epochs', type=int, default=100)
arg('--lr', type=float, default=0.0001)
arg('--workers', type=int, default=2)
arg('--fold', type=int, default=1)
arg('--n-folds', type=int, default=5)
arg('--stratified', action='store_true')
arg('--mode',
choices=[
'train', 'validation', 'predict_valid', 'predict_test',
'predict_all_valid'
],
default='train')
arg('--clean', action='store_true')
arg('--epoch-size', type=int)
arg('--limit', type=int, help='Use only N images for train/valid')
arg('--min-scale', type=float, default=1)
arg('--max-scale', type=float, default=1)
arg('--test-scale', type=float, default=0.5)
args = parser.parse_args()
coords = utils.load_coords()
train_paths, valid_paths = utils.train_valid_split(args, coords)
root = Path(args.root)
model = SSPD()
model = utils.cuda(model)
criterion = SSPDLoss()
if args.mode == 'train':
kwargs = dict(min_scale=args.min_scale, max_scale=args.max_scale)
train_loader, valid_loader = (utils.make_loader(
PointDataset, args, train_paths, coords, **kwargs),
utils.make_loader(PointDataset,
args,
valid_paths,
coords,
deterministic=True,
**kwargs))
if root.exists() and args.clean:
shutil.rmtree(str(root))
root.mkdir(exist_ok=True)
root.joinpath('params.json').write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
utils.train(args,
model,
criterion,
train_loader=train_loader,
valid_loader=valid_loader,
save_predictions=save_predictions)
def main(args):
# Device setting
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Data transformation & augmentation
data_transforms = {
'train':
transforms.Compose([
transforms.Resize((args.resize_pixel, args.resize_pixel)),
transforms.RandomAffine(args.random_affine),
transforms.ColorJitter(brightness=(0.5, 2)),
transforms.RandomResizedCrop(
(args.resize_pixel, args.resize_pixel), scale=(0.85, 1)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
transforms.RandomErasing(p=0.3, scale=(0.01, 0.05))
]),
'valid':
transforms.Compose([
transforms.Resize((args.resize_pixel, args.resize_pixel)),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
}
# Train, valid data split
train_img_list, valid_img_list = train_valid_split(
random_seed=args.random_seed,
data_path=args.data_path,
valid_ratio=args.valid_ratio)
# Custom dataset & dataloader setting
image_datasets = {
'train':
CustomDataset(train_img_list,
isTrain=True,
transform=data_transforms['train']),
'valid':
CustomDataset(valid_img_list,
isTrain=True,
transform=data_transforms['valid'])
}
dataloaders = {
'train':
DataLoader(image_datasets['train'],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers),
'valid':
DataLoader(image_datasets['valid'],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers),
}
# Model Setting
# model = models.mobilenet_v2(pretrained=False, num_classes=10)
# model = conv_model()
if not args.efficientnet_not_use:
model = EfficientNet.from_name(
f'efficientnet-b{args.efficientnet_model_number}')
model._fc = nn.Linear(2304, 26)
else:
model = models.resnext50_32x4d(pretrained=False, num_classes=10)
criterion = nn.CrossEntropyLoss()
optimizer = RAdam(params=filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
lr_step_scheduler = lr_scheduler.StepLR(optimizer,
step_size=args.lr_step_size,
gamma=args.lr_decay_gamma)
model.to(device)
class_names = sorted(list(set(image_datasets['train'].letter)))
class_to_idx = {cl: i for i, cl in enumerate(class_names)}
## Training
# Initialize
best_model_wts = copy.deepcopy(model.state_dict())
best_loss = 9999999999
early_stop = False
# Train
for epoch in range(args.num_epochs):
print('Epoch {}/{}'.format(epoch + 1, args.num_epochs))
if early_stop:
break
for phase in ['train', 'valid']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
start_time = time.time()
# Iterate over data
for inputs, letters, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = torch.tensor([int(class_to_idx[x])
for x in letters]).to(device)
# Zero the parameter gradients
optimizer.zero_grad()
# Track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
# Backward + optimize only if in training phase
if phase == 'train':
loss.backward()
torch_utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
# Statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
# Epoch loss calculate
epoch_loss = running_loss / len(image_datasets[phase])
epoch_acc = running_corrects.double() / len(image_datasets[phase])
if phase == 'valid' and epoch_loss < best_loss:
best_epoch = epoch
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
if phase == 'train' and epoch_loss < 0.001:
early_stop = True
print('Early Stopping!!!')
spend_time = (time.time() - start_time) / 60
print('{} Loss: {:.4f} Acc: {:.4f} Time: {:.3f}min'.format(
phase, epoch_loss, epoch_acc, spend_time))
# Learning rate scheduler
lr_step_scheduler.step()
# Model Saving
model.load_state_dict(best_model_wts)
if not os.path.exists(args.save_path):
os.mkdir(args.save_path)
if not os.path.exists(os.path.join(args.save_path, 'letter')):
os.mkdir(os.path.join(args.save_path, 'letter'))
save_path_ = os.path.join(
os.path.join(args.save_path, 'letter'),
str(datetime.datetime.now())[:-4].replace(' ', '_'))
os.mkdir(save_path_)
print('Best validation loss: {:.4f}'.format(best_loss))
with open(os.path.join(save_path_, 'hyperparameter.json'), 'w') as f:
json.dump(
{
'efficientnet_not_use': args.efficientnet_not_use,
'efficientnet_model_number': args.efficientnet_model_number,
'num_epochs': args.num_epochs,
'resize_pixel': args.resize_pixel,
'random_affine': args.random_affine,
'lr': args.lr,
'random_seed': args.random_seed,
'best_loss': best_loss
}, f)
torch.save(model.state_dict(), os.path.join(save_path_, 'model.pt'))
def preprocessing(args):
#===================================#
#========Data load & Setting========#
#===================================#
# 1) Train data load
train_dat = pd.read_csv(os.path.join(args.data_path, 'train.csv'))
# 2) Preprocessing
def alpha_num(text):
return re.sub(r'[^a-zA-z0-9\s]', '', text)
def remove_stopwords(text):
final_text = []
for i in text.split():
if i.strip().lower() not in stopwords:
final_text.append(i.strip())
return " ".join(final_text)
stopwords = [
"a", "about", "above", "after", "again", "against", "all", "am", "an",
"and", "any", "are", "as", "at", "be", "because", "been", "before",
"being", "below", "between", "both", "but", "by", "could", "did", "do",
"does", "doing", "down", "during", "each", "few", "for", "from",
"further", "had", "has", "have", "having", "he", "he'd", "he'll",
"he's", "her", "here", "here's", "hers", "herself", "him", "himself",
"his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in",
"into", "is", "it", "it's", "its", "itself", "let's", "me", "more",
"most", "my", "myself", "nor", "of", "on", "once", "only", "or",
"other", "ought", "our", "ours", "ourselves", "out", "over", "own",
"same", "she", "she'd", "she'll", "she's", "should", "so", "some",
"such", "than", "that", "that's", "the", "their", "theirs", "them",
"themselves", "then", "there", "there's", "these", "they", "they'd",
"they'll", "they're", "they've", "this", "those", "through", "to",
"too", "under", "until", "up", "very", "was", "we", "we'd", "we'll",
"we're", "we've", "were", "what", "what's", "when", "when's", "where",
"where's", "which", "while", "who", "who's", "whom", "why", "why's",
"with", "would", "you", "you'd", "you'll", "you're", "you've", "your",
"yours", "yourself", "yourselves"
]
# 2) Train valid split
index_list = train_dat['index']
text_list = train_dat['text'].str.lower().apply(alpha_num).apply(
remove_stopwords)
author_list = train_dat['author']
text_, author_, index_ = train_valid_split(
text_list, author_list, index_list, split_percent=args.valid_percent)
# 3) Preprocess path setting
if not os.path.exists(args.save_path):
os.mkdir(args.save_path)
#===================================#
#===========SentencePiece===========#
#===================================#
# 1) Make Korean text to train vocab
with open(f'{args.save_path}/text.txt', 'w') as f:
for text in text_['train']:
f.write(f'{text}\n')
# 2) SentencePiece model training
spm.SentencePieceProcessor()
spm.SentencePieceTrainer.Train(
f'--input={args.save_path}/text.txt --model_prefix={args.save_path}/m_text '
f'--vocab_size={args.vocab_size} --character_coverage=0.995 --model_type=bpe '
f'--split_by_whitespace=true --pad_id={args.pad_idx} --unk_id={args.unk_idx} '
f'--bos_id={args.bos_idx} --eos_id={args.eos_idx}')
# 3) Korean vocabulrary setting
vocab_list = list()
with open(f'{args.save_path}/m_text.vocab') as f:
for line in f:
vocab_list.append(line[:-1].split('\t')[0])
word2id = {w: i for i, w in enumerate(vocab_list)}
# 4) SentencePiece model load
spm_ = spm.SentencePieceProcessor()
spm_.Load(f"{args.save_path}/m_text.model")
# 5) Korean parsing by SentencePiece model
train_text_indices = [[args.bos_idx] + spm_.EncodeAsIds(text) +
[args.eos_idx] for text in text_['train']]
valid_text_indices = [[args.bos_idx] + spm_.EncodeAsIds(text) +
[args.eos_idx] for text in text_['valid']]
#===================================#
#========Test data processing=======#
#===================================#
test_dat = pd.read_csv(os.path.join(args.data_path, 'test_x.csv'))
test_dat['text'] = test_dat['text'].str.lower().apply(alpha_num).apply(
remove_stopwords)
test_text_indices = [[args.bos_idx] + spm_.EncodeAsIds(text) +
[args.eos_idx] for text in test_dat['text']]
test_index_indices = test_dat['index'].tolist()
#===================================#
#==============Saving===============#
#===================================#
print('Parsed sentence saving...')
with open(os.path.join(args.save_path, 'preprocessed.pkl'), 'wb') as f:
pickle.dump(
{
'train_text_indices': train_text_indices,
'valid_text_indices': valid_text_indices,
'train_author_indices': author_['train'],
'valid_author_indices': author_['valid'],
'train_index_indices': index_['train'],
'valid_index_indices': index_['valid'],
'vocab_list': vocab_list,
'word2id': word2id
}, f)
with open(os.path.join(args.save_path, 'test_preprocessed.pkl'),
'wb') as f:
pickle.dump(
{
'test_text_indices': test_text_indices,
'test_index_indices': test_index_indices,
'vocab_list': vocab_list,
'word2id': word2id
}, f)
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('root', help='checkpoint root')
arg('--batch-size', type=int, default=32)
arg('--patch-size', type=int, default=256)
arg('--n-epochs', type=int, default=100)
arg('--lr', type=float, default=0.0001)
arg('--workers', type=int, default=2)
arg('--fold', type=int, default=1)
arg('--bg-weight', type=float, default=1.0, help='background weight')
arg('--dice-weight', type=float, default=0.0)
arg('--n-folds', type=int, default=5)
arg('--stratified', action='store_true')
arg('--mode',
choices=[
'train', 'valid', 'predict_valid', 'predict_test',
'predict_all_valid'
],
default='train')
arg('--model-path',
help='path to model file to use for validation/prediction')
arg('--clean', action='store_true')
arg('--epoch-size', type=int)
arg('--limit', type=int, help='Use only N images for train/valid')
arg('--min-scale', type=float, default=1)
arg('--max-scale', type=float, default=1)
arg('--test-scale', type=float, default=0.5)
arg('--oversample',
type=float,
default=0.0,
help='sample near lion with given probability')
arg('--with-head', action='store_true')
arg('--pred-oddity',
type=int,
help='set to 0/1 to predict even/odd images')
args = parser.parse_args()
coords = utils.load_coords()
train_paths, valid_paths = utils.train_valid_split(args)
root = Path(args.root)
model = UNetWithHead() if args.with_head else UNet()
model = utils.cuda(model)
criterion = Loss(dice_weight=args.dice_weight, bg_weight=args.bg_weight)
loader_kwargs = dict(
min_scale=args.min_scale,
max_scale=args.max_scale,
downscale=args.with_head,
)
if args.mode == 'train':
train_loader, valid_loader = (utils.make_loader(
SegmentationDataset,
args,
train_paths,
coords,
oversample=args.oversample,
**loader_kwargs),
utils.make_loader(SegmentationDataset,
args,
valid_paths,
coords,
deterministic=True,
**loader_kwargs))
if root.exists() and args.clean:
shutil.rmtree(str(root)) # remove dir tree
root.mkdir(exist_ok=True)
root.joinpath('params.json').write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
utils.train(args,
model,
criterion,
train_loader=train_loader,
valid_loader=valid_loader,
save_predictions=save_predictions)
elif args.mode == 'valid':
utils.load_best_model(model, root, args.model_path)
valid_loader = utils.make_loader(SegmentationDataset,
args,
valid_paths,
coords,
deterministic=True,
**loader_kwargs)
utils.validation(model, criterion,
tqdm.tqdm(valid_loader, desc='Validation'))
else:
utils.load_best_model(model, root, args.model_path)
if args.mode in {'predict_valid', 'predict_all_valid'}:
if args.mode == 'predict_all_valid':
# include all paths we did not train on (makes sense only with --limit)
valid_paths = list(
set(valid_paths)
| (set(utils.labeled_paths()) - set(train_paths)))
predict(model,
valid_paths,
out_path=root,
patch_size=args.patch_size,
batch_size=args.batch_size,
min_scale=args.min_scale,
max_scale=args.max_scale,
downsampled=args.with_head)
elif args.mode == 'predict_test':
out_path = root.joinpath('test')
out_path.mkdir(exist_ok=True)
predicted = {p.stem.split('-')[0] for p in out_path.glob('*.npy')}
test_paths = [
p for p in utils.DATA_ROOT.joinpath('Test').glob('*.png')
if p.stem not in predicted
]
if args.pred_oddity is not None:
assert args.pred_oddity in {0, 1}
test_paths = [
p for p in test_paths
if int(p.stem) % 2 == args.pred_oddity
]
predict(model,
test_paths,
out_path,
patch_size=args.patch_size,
batch_size=args.batch_size,
test_scale=args.test_scale,
is_test=True,
downsampled=args.with_head)
else:
parser.error('Unexpected mode {}'.format(args.mode))
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('root', help='checkpoint root')
arg('out_path', help='path to UNet features', type=Path)
arg('--batch-size', type=int, default=32)
arg('--patch-size', type=int, default=160)
arg('--offset', type=int, default=6)
arg('--n-epochs', type=int, default=100)
arg('--lr', type=float, default=0.0001)
arg('--workers', type=int, default=2)
arg('--fold', type=int, default=1)
arg('--n-folds', type=int, default=5)
arg('--stratified', action='store_true')
arg('--mode',
choices=[
'train', 'valid', 'predict_valid', 'predict_test',
'predict_all_valid'
],
default='train')
arg('--model-path',
help='path to model file to use for validation/prediction')
arg('--clean', action='store_true')
arg('--epoch-size', type=int)
arg('--limit', type=int, help='Use only N images for train/valid')
arg('--min-scale', type=float, default=1)
arg('--max-scale', type=float, default=1)
arg('--test-scale', type=float, default=0.5)
arg('--pred-oddity',
type=int,
help='set to 0/1 to predict even/odd images')
args = parser.parse_args()
coords = utils.load_coords()
train_paths, valid_paths = utils.train_valid_split(args, coords)
root = Path(args.root)
model = VGGModel(args.patch_size)
model = utils.cuda(model)
criterion = nn.CrossEntropyLoss()
loader_kwargs = dict(min_scale=args.min_scale,
max_scale=args.max_scale,
offset=args.offset)
if args.mode == 'train':
train_loader, valid_loader = (utils.make_loader(
ClassificationDataset, args, train_paths, coords, **loader_kwargs),
utils.make_loader(ClassificationDataset,
args,
valid_paths,
coords,
deterministic=True,
**loader_kwargs))
if root.exists() and args.clean:
shutil.rmtree(str(root))
root.mkdir(exist_ok=True)
root.joinpath('params.json').write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
utils.train(
args,
model,
criterion,
train_loader=train_loader,
valid_loader=valid_loader,
save_predictions=save_predictions,
is_classification=True,
make_optimizer=lambda lr: SGD([
{
'params': model.features.parameters(),
'lr': lr
},
{
'params': model.classifier.parameters(),
'lr': lr
},
],
nesterov=True,
momentum=0.9),
)
elif args.mode == 'valid':
utils.load_best_model(model, root, args.model_path)
valid_loader = utils.make_loader(ClassificationDataset,
args,
valid_paths,
coords,
deterministic=True,
**loader_kwargs)
utils.validation(model,
criterion,
tqdm.tqdm(valid_loader, desc='Validation'),
is_classification=True)
else:
utils.load_best_model(model, root, args.model_path)
if args.mode in {'predict_valid', 'predict_all_valid'}:
if args.mode == 'predict_all_valid':
# include all paths we did not train on (makes sense only with --limit)
valid_paths = list(
set(valid_paths)
| (set(utils.labeled_paths()) - set(train_paths)))
predict(model,
valid_paths,
out_path=args.out_path,
patch_size=args.patch_size,
batch_size=args.batch_size,
min_scale=args.min_scale,
max_scale=args.max_scale)
elif args.mode == 'predict_test':
assert False # FIXME - use out_path too
out_path = root.joinpath('test')
out_path.mkdir(exist_ok=True)
predicted = {p.stem.split('-')[0] for p in out_path.glob('*.npy')}
test_paths = [
p for p in utils.DATA_ROOT.joinpath('Test').glob('*.jpg')
if p.stem not in predicted
]
if args.pred_oddity is not None:
assert args.pred_oddity in {0, 1}
test_paths = [
p for p in test_paths
if int(p.stem) % 2 == args.pred_oddity
]
predict(model,
test_paths,
out_path,
patch_size=args.patch_size,
batch_size=args.batch_size,
test_scale=args.test_scale,
is_test=True)
else:
parser.error('Unexpected mode {}'.format(args.mode))
def load_dataset(dataset, train_size, valid_size, test_size):
"""Load the dataset passed in argument with the corresponding sizes for the training, validation and testing set."""
if dataset == 'mnist_012':
root = './data/mnist'
num_classes = 3
trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.ToTensor(), transforms.Normalize(mean=MNIST_MEAN, std=MNIST_STD)])
train_valid_set = datasets.MNIST(root=root, train=True, transform=trans)
test_set = datasets.MNIST(root=root, train=False, transform=trans)
train_valid_set = MNIST_bis(dataset=train_valid_set, size=train_size+valid_size, digits_to_keep=[0,1,2])
test_set = MNIST_bis(dataset=test_set, size=test_size, digits_to_keep=[0,1,2])
train_sampler, valid_sampler = train_valid_split(dataset=train_valid_set, train_size=train_size)
train_loader = DataLoader(dataset=train_valid_set, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=4, pin_memory=True, drop_last=True)
valid_loader = DataLoader(dataset=train_valid_set, batch_size=BATCH_SIZE, sampler=valid_sampler, num_workers=4, pin_memory=True, drop_last=True)
test_loader = DataLoader(dataset=test_set, batch_size=BATCH_SIZE, num_workers=4, pin_memory=True, drop_last=True)
elif dataset == 'mnist_rot':
root = './data/mnist'
num_classes = 9
train_trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize((26,26)), transforms.ToTensor(), transforms.Normalize(mean=MNIST_MEAN, std=MNIST_STD)])
test_trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize((26,26)), transforms.RandomRotation((0,360)), transforms.ToTensor(), transforms.Normalize(mean=MNIST_MEAN, std=MNIST_STD)])
train_valid_set = datasets.MNIST(root=root, train=True, transform=train_trans)
test_set = datasets.MNIST(root=root, train=False, transform=test_trans)
train_valid_set_bis = MNIST_bis(dataset=train_valid_set, size=train_size+valid_size, digits_to_keep=[0,1,2,3,4,5,6,7,8])
test_set = MNIST_bis(dataset=test_set, size=test_size, digits_to_keep=[0,1,2,3,4,5,6,7,8])
train_sampler, valid_sampler = train_valid_split(dataset=train_valid_set_bis, train_size=train_size)
train_loader = DataLoader(dataset=train_valid_set_bis, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=4, pin_memory=True, drop_last=True)
valid_loader = DataLoader(dataset=train_valid_set_bis, batch_size=BATCH_SIZE, sampler=valid_sampler, num_workers=4, pin_memory=True, drop_last=True)
test_loader = DataLoader(dataset=test_set, batch_size=BATCH_SIZE, num_workers=4, pin_memory=True, drop_last=True)
elif dataset == 'mnist_trans':
root = './data/mnist'
num_classes = 9
train_trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize((26,26)), transforms.ToTensor(), transforms.Normalize(mean=MNIST_MEAN, std=MNIST_STD)])
test_trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize((26,26)), RandomTranslation(horizontal=6, vertical=6), transforms.ToTensor(), transforms.Normalize(mean=MNIST_MEAN, std=MNIST_STD)])
train_valid_set = datasets.MNIST(root=root, train=True, transform=train_trans)
test_set = datasets.MNIST(root=root, train=False, transform=test_trans)
train_valid_set_bis = MNIST_bis(dataset=train_valid_set, size=train_size+valid_size, digits_to_keep=[0,1,2,3,4,5,6,7,8])
test_set = MNIST_bis(dataset=test_set, size=test_size, digits_to_keep=[0,1,2,3,4,5,6,7,8])
train_sampler, valid_sampler = train_valid_split(dataset=train_valid_set_bis, train_size=train_size)
train_loader = DataLoader(dataset=train_valid_set_bis, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=4, pin_memory=True, drop_last=True)
valid_loader = DataLoader(dataset=train_valid_set_bis, batch_size=BATCH_SIZE, sampler=valid_sampler, num_workers=4, pin_memory=True, drop_last=True)
test_loader = DataLoader(dataset=test_set, batch_size=BATCH_SIZE, num_workers=4, pin_memory=True, drop_last=True)
elif dataset == 'eth80':
root = './data/eth80'
num_classes = 8
trans = transforms.Compose([transforms.Grayscale(num_output_channels=1), transforms.Resize((50,50)), transforms.ToTensor(), transforms.Normalize(mean=ETH80_MEAN, std=ETH80_STD)])
complete_set = datasets.ImageFolder(root=root, transform=trans)
class_names = complete_set.classes
train_sampler, valid_sampler, test_sampler = train_valid_test_split(dataset=complete_set, train_size=train_size, valid_size=valid_size)
train_loader = DataLoader(dataset=complete_set, batch_size=BATCH_SIZE, sampler=train_sampler, num_workers=4, pin_memory=True, drop_last=True)
valid_loader = DataLoader(dataset=complete_set, batch_size=BATCH_SIZE, sampler=valid_sampler, num_workers=4, pin_memory=True, drop_last=True)
test_loader = DataLoader(dataset=complete_set, batch_size=BATCH_SIZE, sampler=test_sampler, num_workers=4, pin_memory=True, drop_last=True)
else:
raise ValueError('Specified dataset does not exist.')
logger.debug('Class frequency train loader: {} validation loader: {} test loader: {}'.format(
count_class_freq(train_loader, num_classes),count_class_freq(valid_loader, num_classes), count_class_freq(test_loader, num_classes))
)
logging.info('Loaded {} dataset with the split {}-{}-{} for the [train]-[valid]-[test] setup.'.format(dataset, len(train_loader)*BATCH_SIZE, len(valid_loader)*BATCH_SIZE, len(test_loader)*BATCH_SIZE))
return train_loader, valid_loader, test_loader, get_dim(train_loader)
def train(args):
# Device setting
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Data transformation & augmentation
data_transforms = {
'train':
A.Compose([
A.Resize(args.resize_pixel, args.resize_pixel, always_apply=True),
A.RandomRotate90(),
A.Flip(),
A.Transpose(),
A.OneOf([
A.IAAAdditiveGaussianNoise(),
A.GaussNoise(),
], p=0.2),
A.OneOf([
A.MotionBlur(p=.2),
A.MedianBlur(blur_limit=3, p=0.1),
A.Blur(blur_limit=3, p=0.1),
],
p=0.2),
A.ShiftScaleRotate(shift_limit=0.0625,
scale_limit=0.2,
rotate_limit=45,
p=0.2),
A.OneOf([
A.OpticalDistortion(p=0.3),
A.GridDistortion(p=.1),
A.IAAPiecewiseAffine(p=0.3),
],
p=0.2),
A.OneOf([
A.CLAHE(clip_limit=2),
A.IAASharpen(),
A.IAAEmboss(),
A.RandomBrightnessContrast(),
],
p=0.3),
A.HueSaturationValue(p=0.3),
# A.pytorch.transforms.ToTensor(),
A.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
]),
'valid':
A.Compose([
A.Resize(args.resize_pixel, args.resize_pixel, always_apply=True),
# A.pytorch.ToTensor(),
A.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
}
# Train, valid data split
train_img_list, valid_img_list = train_valid_split(
random_seed=args.random_seed,
data_path=args.data_path,
valid_ratio=args.valid_ratio)
# Custom dataset & dataloader setting
image_datasets = {
'train':
CustomDataset(train_img_list,
args.data_path,
transform=data_transforms['train'],
isTest=False),
'valid':
CustomDataset(valid_img_list,
args.data_path,
transform=data_transforms['valid'],
isTest=False)
}
dataloaders = {
'train':
DataLoader(image_datasets['train'],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers),
'valid':
DataLoader(image_datasets['valid'],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers),
}
# Training setting
model = EfficientNet.from_pretrained(
f'efficientnet-b{args.efficientnet_model_number}', num_classes=1049)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr)
scheduler = WarmupLinearSchedule(
optimizer,
warmup_steps=len(dataloaders['train']) * 3,
t_total=len(dataloaders['train']) * args.num_epochs)
model.to(device)
# Initialize
best_model_wts = copy.deepcopy(model.state_dict())
best_loss = 9999999999
early_stop = False
# Train start
for epoch in range(args.num_epochs):
start_time_e = time.time()
print('[Epoch {}/{}]'.format(epoch + 1, args.num_epochs))
if early_stop:
print('Early Stopping!!!')
break
for phase in ['train', 'valid']:
if phase == 'train':
model.train()
else:
model.eval()
running_loss = 0.0
running_corrects = 0
start_time = time.time()
# Iterate over data
for i, (inputs, labels) in enumerate(dataloaders[phase]):
inputs = inputs.to(device)
labels = torch.tensor([int(x) for x in labels]).to(device)
# Zero the parameter gradients
optimizer.zero_grad()
# Track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
# If phase train, then backward loss and step optimizer and scheduler
if phase == 'train':
loss.backward()
torch_utils.clip_grad_norm_(model.parameters(),
args.max_grad_norm)
optimizer.step()
scheduler.step()
# Print loss value only training
if i == 0 or freq == args.print_freq or i == len(
dataloaders['train']):
total_loss = loss.item()
print(
"[Epoch:%d][%d/%d] train_loss:%5.3f | learning_rate:%3.8f | spend_time:%5.2fmin"
% (epoch + 1, i, len(dataloaders['train']),
total_loss, optimizer.param_groups[0]['lr'],
(time.time() - start_time_e) / 60))
freq = 0
freq += 1
# Statistics
running_loss += loss.item() * inputs.size(0)
if phase == 'valid':
val_loss = running_loss
running_corrects += torch.sum(preds == labels.data)
# Epoch loss calculate
epoch_loss = running_loss / len(image_datasets[phase])
epoch_acc = running_corrects.double() / len(image_datasets[phase])
if phase == 'valid' and epoch_loss < best_loss:
best_epoch = epoch
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
if phase == 'train' and epoch_loss < 0.001:
early_stop = True
spend_time = (time.time() - start_time) / 60
print('{} Loss: {:.4f} Acc: {:.4f} Time: {:.3f}min'.format(
phase, epoch_loss, epoch_acc, spend_time))
# Model Saving
model.load_state_dict(best_model_wts)
if not os.path.exists(args.save_path):
os.mkdir(args.save_path)
best_loss = round(best_loss, 4)
save_path_ = os.path.join(
args.save_path,
str(datetime.datetime.now())[:10] + f'_{best_loss}')
os.mkdir(save_path_)
print('Best validation loss: {:.4f}'.format(best_loss))
with open(os.path.join(save_path_, 'hyperparameter.json'), 'w') as f:
json.dump(
{
'efficientnet_not_use': args.efficientnet_not_use,
'efficientnet_model_number': args.efficientnet_model_number,
'num_epochs': args.num_epochs,
'resize_pixel': args.resize_pixel,
'random_affine': args.random_affine,
'lr': args.lr,
'random_seed': args.random_seed,
'best_loss': best_loss
}, f)
torch.save(model.state_dict(), os.path.join(save_path_, 'model.pt'))
'scheduler_lr': scheduler._rate,
'scheduler_step': scheduler._step
}
if not os.path.isdir("./checkpoint"):
os.mkdir("./checkpoint")
torch.save(
checkpoint, './checkpoint/weights_{}_acc_{:.2f}.pth'.format(
e + 1, test_acc * 100))
return train_losses, test_losses
if __name__ == '__main__':
RESUME = True
hparams = HParams()
metadata_filename = './training_data/train.txt'
metadata, training_idx, validation_idx = train_valid_split(
metadata_filename, hparams, test_size=0.05, seed=0)
train_dataset = SpeechDataset(metadata_filename,
list(np.array(metadata)[training_idx]),
hparams)
test_dataset = SpeechDataset(metadata_filename,
list(np.array(metadata)[validation_idx]),
hparams)
train_loader = DataLoader(train_dataset,
1,
True,
num_workers=4,
pin_memory=False)
test_loader = DataLoader(test_dataset,
1,
True,
from utils.preprocess import zscore_normalization, fill_NaN_w_mean
from utils.feature_selection import LassoSelection, FscoreSelection, MISelection
from utils.oversampling import oversample_ROSE, oversample_SMOTE
from utils.export_tools import export_result_to_excel
# Seed
np.random.seed(0)
random.seed(0)
# Parse argument
opt = parse_option(print_option=True)
# Load Data
print("Loading Data...")
data_df = load_data(opt)
train_df_org, valid_df_org = train_valid_split(data_df, opt)
# Pre-processing : Z-score Normalization & Remove NaN values
print("Pre-processing...")
train_df_z, valid_df_z = zscore_normalization(train_df_org, valid_df_org, opt)
train_df_filled, valid_df_filled = fill_NaN_w_mean(train_df_z, valid_df_z)
# Feature Selection (Utilize 'Feature selection method' : 'Dataset with selected features' dictionary)
print("Feature Selection...")
dataset_dict_fs = dict()
dataset_dict_fs['LASSO'] = LassoSelection(train_df_filled, valid_df_filled,
opt)
dataset_dict_fs['Fscore'] = FscoreSelection(train_df_filled, valid_df_filled,
opt)
dataset_dict_fs['MI'] = MISelection(train_df_filled, valid_df_filled, opt)
