from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import Normalize
import albumentations
from albumentations.pytorch import ToTensor
import numpy as np
imagenet_normalize = Normalize( # imagenet mean + standard deviation
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
detection_base = albumentations.Compose([ToTensor()])
detection_augment = albumentations.Compose([
albumentations.HorizontalFlip(),
albumentations.OneOf(
[
albumentations.RandomContrast(),
albumentations.RandomGamma(),
albumentations.RandomBrightness(),
],
p=0.3,
),
albumentations.OneOf(
[
albumentations.ElasticTransform(
alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
albumentations.GridDistortion(),
albumentations.OpticalDistortion(distort_limit=2, shift_limit=0.5),
],
p=0.3,
),
def train(**kwargs):
if len(kwargs) == 0:
parser = argparse.ArgumentParser()
parser.add_argument("--config",
default="./vanilla.json",
type=str,
help="path to the config file")
args = parser.parse_args()
args = Config(args.config)
else:
args = Config(kwargs.get("config", "./vanilla.json"))
print(json.dumps(args.__dict__, indent=4))
get_dataset_with_arg = {
"tiny_imagenet": food.datasets.TinyImagenet,
"cifar_100": food.datasets.CIFAR_100,
"cifar_10": food.datasets.CIFAR_10,
"mnist": MNIST
}
if not args.keep_logs:
try:
shutil.rmtree(args.logdir)
except FileNotFoundError:
pass
if args.dataset == 'cifar_10':
A_tr = 0.9500 # Training accuracy of CIFAR-10 baseline model
elif args.dataset == 'cifar_100':
A_tr = 0.8108
elif args.dataset == 'mnist':
A_tr = 0.9885
batch_size = args.batch_size
batch_size = kwargs.get('batch_size', batch_size)
model = args.model.lower()
model = kwargs.get('model', model).lower()
dataset = args.dataset.lower()
dataset = kwargs.get('dataset', dataset).lower()
epochs = args.epochs
epochs = kwargs.get('epochs', epochs)
test_b = kwargs.get('test', False)
ds_class = get_dataset_with_arg[dataset.lower()]
train_transforms = Compose([Normalize((0.1307, ), (0.3081, )), ToTensor()])
val_transforms = train_transforms
train_dataset = ds_class(args.data_path,
mode="train",
task=args.task.lower(),
transform=train_transforms)
val_dataset = ds_class(args.data_path,
mode="val",
task=args.task.lower(),
transform=val_transforms)
ood_dataset_train = FashionMNIST(args.data_path,
mode="train",
task="vanilla",
transform=val_transforms)
ood_dataset_val = FashionMNIST(args.data_path,
mode="val",
task="vanilla",
transform=val_transforms)
model = MnistNet()
print(model)
print(model.__class__.__name__)
print("Total number of model's parameters: ",
np.sum([p.numel() for p in model.parameters() if p.requires_grad]))
train_dataloader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True)
val_dataloader = DataLoader(val_dataset,
batch_size=batch_size,
shuffle=False,
drop_last=False)
ood_dataloader_train = DataLoader(ood_dataset_train,
batch_size=batch_size,
shuffle=False,
drop_last=False)
ood_dataloader_val = DataLoader(ood_dataset_val,
batch_size=batch_size,
shuffle=False,
drop_last=False)
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model.to(device)
criterion = nn.CrossEntropyLoss()
# Optimizer
optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=0.5)
# Learning Rate
def cosine_annealing(step, total_steps, lr_max, lr_min):
return lr_min + (lr_max - lr_min) * 0.5 * (
1 + np.cos(step / total_steps * np.pi))
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer,
lr_lambda=lambda step: cosine_annealing(
step,
args.epochs * len(train_dataloader),
1, # since lr_lambda computes multiplicative factor
1e-6 / args.lr))
resume_epoch = 0
if args.resume is not None:
state_dict = torch.load(args.resume, map_location=device)
# state_dict = state_dict["state_dict"]
old_state_dict = model.state_dict()
for key in state_dict:
old_state_dict[key] = state_dict[key]
model.load_state_dict(old_state_dict)
os.makedirs(os.path.join(args.logdir, "train_logs"), exist_ok=True)
os.makedirs(os.path.join(args.logdir, "val_logs"), exist_ok=True)
# train_writer = SummaryWriter(os.path.join(args.logdir, "train_logs"))
val_writer = SummaryWriter(os.path.join(args.logdir, "val_logs"))
global_step = 0
for epoch in range(resume_epoch, epochs):
print(f"Training, epoch {epoch + 1}")
model.train()
val_loss, val_acc = evaluate(model,
val_dataloader,
ood_dataloader_val,
criterion,
device,
val_writer,
T=args.temperature,
eps=args.eps)
exit(0)
for (images, labels), (ood_images, ood_labels) in tqdm.tqdm(
zip(train_dataloader, ood_dataloader_train)):
images, labels = images.to(device), labels.to(device).long()
ood_images = ood_images.to(device)
optimizer.zero_grad()
# predict in-distribution probabilities
logits_in = model.forward(images)
probabilites_in = F.softmax(logits_in)
loss = criterion(logits_in, labels)
max_probs_in, _ = probabilites_in.max(1)
# predict ood-probabilities
logits_out = model.forward(ood_images)
probabilites_out = F.softmax(logits_out)
# calculate regularization
loss += args.lambda1 * torch.sum((max_probs_in - A_tr)**2)
loss += args.lambda2 * \
torch.sum(torch.abs(probabilites_out -
1 / train_dataset.n_classes))
# predictions = logits.argmax(dim=1)
# accuracy_t = torch.mean((predictions == labels).float()).item()
if global_step % args.log_each == 0:
# TODO: train logging for tensorboard
# train_writer.add_scalar("Loss_BCE", loss, global_step=global_step)
# train_writer.add_scalar("Accuracy", accuracy_t, global_step=global_step)
# train_writer.add_scalar("Learning_rate", scheduler.get_lr()[-1], global_step=global_step)
# log_dict_with_writer(labels, logits, train_writer, global_step=global_step)
pass
loss.backward()
optimizer.step()
global_step += 1
print("Validating...")
val_loss, val_acc = evaluate(model,
val_dataloader,
ood_dataloader_val,
criterion,
device,
val_writer,
T=args.temperature,
eps=args.temperature)
val_writer.add_scalar("Loss_BCE", val_loss, global_step=global_step)
val_writer.add_scalar("Accuracy", val_acc, global_step=global_step)
if epoch % args.checkpoint_each == 0:
print("Saving checkpoint...")
with open(
os.path.join(args.checkpoints_dir,
f"epoch_{epoch}_{global_step}.pt"),
"wb") as f:
torch.save(
{
"model_state_dict": model.state_dict(),
"epoch": epoch,
"scheduler_state_dict": scheduler.state_dict(),
"optimizer_state_dict": optimizer.state_dict()
}, f)
scheduler.step(val_acc)
if test_b:
return logits, loss, predictions, val_loss, val_acc
png = np.array(png)
train = train.set_index('Image').loc[png].reset_index()
# get fold
valdf = train[train['fold'] == fold].reset_index(drop=True)
trndf = train[train['fold'] != fold].reset_index(drop=True)
# Data loaders
transform_train = Compose([
HorizontalFlip(p=0.5),
ShiftScaleRotate(shift_limit=0.1,
scale_limit=0.01,
rotate_limit=30,
p=0.7,
border_mode=cv2.BORDER_REPLICATE),
ToTensor()
])
transform_test = Compose([ToTensor()])
trndataset = IntracranialDataset(trndf,
path=dir_train_img,
transform=transform_train,
labels=True)
valdataset = IntracranialDataset(valdf,
path=dir_train_img,
transform=transform_test,
labels=False)
tstdataset = IntracranialDataset(test,
path=dir_test_img,
transform=transform_test,
expand=True)
test['Image'] = 'ID_' + test['Image']
test = test[['Image', 'Label']]
test.drop_duplicates(inplace=True)
test.to_csv('test.csv', index=False)
# Data loaders
transform_train = Compose([
CenterCrop(200, 200),
#Resize(224, 224),
HorizontalFlip(),
RandomBrightnessContrast(),
ShiftScaleRotate(),
ToTensor()
])
transform_test = Compose([
CenterCrop(200, 200),
#Resize(224, 224),
ToTensor()
])
train_dataset = IntracranialDataset(csv_file='train.csv',
data_dir=dir_train_img,
transform=transform_train,
labels=True)
test_dataset = IntracranialDataset(csv_file='test.csv',
data_dir=dir_test_img,
# ##############################
mean = (0.485, 0.456, 0.406)
std = (0.229, 0.224, 0.225)
train_transforms = A.Compose([
A.RandomScale(scale_limit=(0.0, 1.5),
interpolation=cv2.INTER_LINEAR,
p=1.0),
A.PadIfNeeded(val_img_size, val_img_size, border_mode=cv2.BORDER_CONSTANT),
A.RandomCrop(train_img_size, train_img_size),
A.HorizontalFlip(),
A.Blur(blur_limit=3),
A.Normalize(mean=mean, std=std),
ignore_mask_boundaries,
ToTensor(),
])
val_transforms = A.Compose([
A.PadIfNeeded(val_img_size, val_img_size, border_mode=cv2.BORDER_CONSTANT),
A.Normalize(mean=mean, std=std),
ignore_mask_boundaries,
ToTensor(),
])
train_loader, val_loader, train_eval_loader = get_train_val_loaders(
root_path=data_path,
train_transforms=train_transforms,
val_transforms=val_transforms,
batch_size=batch_size,
num_workers=num_workers,
def test_torch_to_tensor_augmentations(image, mask):
aug = ToTensor()
data = aug(image=image, mask=mask, force_apply=True)
assert data['image'].dtype == torch.float32
assert data['mask'].dtype == torch.float32
def getAlbumTransformation(transformList):
transformList.append(ToTensor())
data_transforms = Compose(transformList)
return lambda x: data_transforms(image=np.array(x))['image']
def post_transforms():
return Compose([Normalize(), ToTensor()])
def make_prediction(
device_num: int,
equalization: bool = False,
normalization: bool = False,
):
device = torch.device(f'cuda:{device_num}')
trn_lbl_df = load_trn_lbl_df()
trn_hdr_df = pd.read_pickle(TRN_HDR_DF_PATH)
tst_hdr_df = pd.read_pickle(TST_HDR_DF_PATH)
trn_df = pd.concat([trn_hdr_df, trn_lbl_df], axis=1, join='inner')
trn_outlier_mask, tst_outlier_mask = get_outlier(trn_df, tst_hdr_df)
valid_trn_df = trn_df[~trn_outlier_mask]
trn_dset_kwargs = {
'training': True,
'dataframe': valid_trn_df,
'window_ranges': DEFAULT_WINDOW_RANGES,
'equalization': equalization,
'regularize_dim': 512,
'low_memory': True,
}
tst_dset_kwargs = {
'training': False,
'dataframe': tst_hdr_df,
'window_ranges': DEFAULT_WINDOW_RANGES,
'equalization': equalization,
'regularize_dim': 512,
'low_memory': True,
}
if normalization:
channel_avgs, channel_stds, nan_sample_ids = \
normalize_dset(trn_dset_kwargs, num_workers=-1, batch_size=128)
valid_trn_df.drop(nan_sample_ids, inplace=True)
trn_transform = Compose([
Resize(IMG_DIM, IMG_DIM),
Normalize(mean=channel_avgs, std=channel_stds),
HorizontalFlip(),
RandomBrightnessContrast(),
ShiftScaleRotate(),
ToTensor()
])
tst_transform = Compose([
Resize(IMG_DIM, IMG_DIM),
Normalize(mean=channel_avgs, std=channel_stds),
ToTensor()
])
else:
trn_transform = Compose([
Resize(IMG_DIM, IMG_DIM),
HorizontalFlip(),
RandomBrightnessContrast(),
ShiftScaleRotate(),
ToTensor()
])
tst_transform = Compose([Resize(IMG_DIM, IMG_DIM), ToTensor()])
trn_dset = ICHDataset(**trn_dset_kwargs, transform=trn_transform)
tst_dset = ICHDataset(**tst_dset_kwargs, transform=tst_transform)
dldr_kwargs = {
'batch_size': 32,
'num_workers': 32,
'pin_memory': True,
'timeout': 1000,
}
trn_dldr = DataLoader(trn_dset, **dldr_kwargs)
tst_dldr = DataLoader(tst_dset, **dldr_kwargs)
model = torch.hub.load(
'facebookresearch/WSL-Images',
f'resnext101_32x{NUM_CARDINALITY}d_wsl',
).to(device)
model.fc = torch.nn.Linear(2048, len(DIAGNOSIS)).to(device)
criterion = torch.nn.BCEWithLogitsLoss(
weight=torch.FloatTensor([2, 1, 1, 1, 1, 1]).to(device))
optim = torch.optim.Adam(params=model.parameters(), lr=2e-5)
amp.initialize(model, optim, opt_level='O1')
for epoch in range(NUM_EPOCHS):
print('Epoch {}/{}'.format(epoch, NUM_EPOCHS - 1))
print('-' * 10)
# model.train()
# tr_loss = 0
#
# trn_iter = tqdm(trn_dldr, desc="Iteration")
#
# for step, batch in enumerate(trn_iter):
#
# inputs = batch["image"]
# labels = batch["labels"]
#
# inputs = inputs.to(device, dtype=torch.float)
# labels = labels.to(device, dtype=torch.float)
#
# outputs = model(inputs)
# loss = criterion(outputs, labels)
#
# loss.backward()
#
# tr_loss += loss.item()
#
# optim.step()
# optim.zero_grad()
#
# if epoch == 1 and step > 6000:
# epoch_loss = tr_loss / 6000
# print('Training Loss: {:.4f}'.format(epoch_loss))
# break
#
# epoch_loss = tr_loss / len(trn_dldr)
# print('Training Loss: {:.4f}'.format(epoch_loss))
with torch.no_grad():
model.eval()
ids = []
predictions = []
tst_iter = tqdm(tst_dldr, desc="Iteration")
for step, batch in enumerate(tst_iter):
inputs = batch['image']
ids.extend(batch['id'])
inputs = inputs.to(device, dtype=torch.float)
# labels = labels.to(device, dtype=torch.float)
predictions.extend(torch.sigmoid(model(inputs)).tolist())
pred_df = pd.DataFrame(predictions, columns=DIAGNOSIS, index=ids)
pred_df.index.name = 'ID'
masked_pred_df = pred_df.copy(deep=True)
masked_pred_df.loc[tst_outlier_mask] = \
[[0, 0, 0, 0, 0, 0]] * tst_outlier_mask.sum()
print(pred_df.head())
print(masked_pred_df.head())
tst_lbl_df_to_submission_csv(
pred_df, f'./e_{equalization}_n_{normalization}_{epoch+1}.csv')
tst_lbl_df_to_submission_csv(
masked_pred_df,
f'./masked_e_{equalization}_n_{normalization}_{epoch+1}.csv')
transforms.RandomHorizontalFlip(),
transforms.CenterCrop(32),
transforms.ToTensor(),
transforms.Normalize(*stats)
])
train_transform = A.Compose([ # cutout, HorizontalFlip
albtransforms.RandomCrop(28, 28, always_apply=False, p=1.0),
albtransforms.Rotate(30, p=0.1),
albtransforms.Cutout(num_holes=8,
max_h_size=8,
max_w_size=8,
always_apply=False,
p=0.1),
albtransforms.HorizontalFlip(1.0),
A.Normalize(*stats),
ToTensor()
])
test_transform = A.Compose([A.Normalize(*stats), ToTensor()])
# loaded only when loaddata() invoked
trainset = None
trainloader = None
testset = None
testloader = None
def __custom_getitem__(self, index):
image, label = self.data[index], self.targets[index]
if self.transform is not None:
transformed = self.transform(image=image)
def get_poor_man_data_aug(mean, standard_deviation):
return [
A.Normalize(mean=mean, std=standard_deviation),
ToTensor()
]
def __init__(self):
self.master = img_trans = albumentations.Compose([
albumentations.Normalize(mean=[0.798, 0.621, 0.841],
std=[0.125, 0.228, 0.089]),
ToTensor(),
])
#return {'image': img, 'labels': labels}
return (img, labels)
else:
#return {'image': img}
return img
# In[6]:
transform_train = Compose([
Resize(456, 456),
ToFloat(max_value=127, always_apply=True),
ShiftScaleRotate(),
ToTensor()
])
transform_valid = Compose(
[Resize(456, 456),
ToFloat(max_value=127, always_apply=True),
ToTensor()])
transform_test = Compose(
[Resize(456, 456),
ToFloat(max_value=127, always_apply=True),
ToTensor()])
train_dataset = IntracranialDataset(
csv_file='/mnt/disks/mount_point/train.csv',
path='/mnt/disks/mount_point/stage_1_train_images_jpg',
def get_transforms(phase_config):
list_transforms = []
if phase_config.Resize.p > 0:
list_transforms.append(
Resize(phase_config.Resize.height, phase_config.Resize.width, p=1))
if phase_config.HorizontalFlip:
list_transforms.append(HorizontalFlip())
if phase_config.VerticalFlip:
list_transforms.append(VerticalFlip())
if phase_config.RandomCropScale:
if phase_config.Resize.p > 0:
height = phase_config.Resize.height
width = phase_config.Resize.width
else:
height = HEIGHT
width = WIDTH
list_transforms.append(
RandomSizedCrop(min_max_height=(int(height * 0.90), height),
height=height,
width=width,
w2h_ratio=width / height))
if phase_config.ShiftScaleRotate:
list_transforms.append(ShiftScaleRotate(p=1))
if phase_config.RandomCrop.p > 0:
list_transforms.append(
RandomCrop(phase_config.RandomCrop.height,
phase_config.RandomCrop.width,
p=1))
if phase_config.Noise:
list_transforms.append(
OneOf([
GaussNoise(),
IAAAdditiveGaussianNoise(),
], p=0.5), )
if phase_config.Contrast:
list_transforms.append(
OneOf([
RandomContrast(0.5),
RandomGamma(),
RandomBrightness(),
],
p=0.5), )
if phase_config.Blur:
list_transforms.append(
OneOf([
MotionBlur(p=.2),
MedianBlur(blur_limit=3, p=0.1),
Blur(blur_limit=3, p=0.1),
],
p=0.5))
if phase_config.Distort:
list_transforms.append(
OneOf([
OpticalDistortion(p=0.3),
GridDistortion(p=.1),
IAAPiecewiseAffine(p=0.3),
],
p=0.5))
if phase_config.Cutout.num_holes > 0:
num_holes = phase_config.Cutout.num_holes
hole_size = phase_config.Cutout.hole_size
list_transforms.append(Cutout(num_holes, hole_size))
list_transforms.extend([
Normalize(mean=phase_config.mean, std=phase_config.std, p=1),
ToTensor(),
])
return Compose(list_transforms)
# image_id = 'c14c1e300'
# bboxes = df.loc[df['image_id'] == image_id, ['x', 'y', 'w', 'h']].values
# image_read('./input/train/', image_id, bboxes)
sys.exit(0)
# image_id = 'c14c1e300'
# bboxes = df.loc[df['image_id'] == image_id, ['x', 'y', 'w', 'h']].values
# image_read('./input/train/', image_id, bboxes)
# sys.exit(0)
image_ids = df['image_id'].unique()
# sys.exit(0)
image_dir = './input/train/'
transforms = {'train': A.Compose([ToTensor()])}
trainset = TrainDataset(df,
image_dir,
image_ids,
transforms=transforms['train'])
def collate_fn(batch):
return tuple(zip(*(batch)))
train_loader = DataLoader(trainset,
batch_size=2,
shuffle=True,
num_workers=4,
collate_fn=collate_fn)
def albumentations_transforms(
crop_size,
shorter_side,
low_scale,
high_scale,
img_mean,
img_std,
img_scale,
ignore_label,
num_stages,
dataset_type,
):
from albumentations import (
Normalize,
HorizontalFlip,
RandomRotate90, # my addition
RandomBrightnessContrast, # my addition
CLAHE, # my addition
RandomGamma, # my addition
ElasticTransform, # my addition
GridDistortion, # my addition
MotionBlur, # my addition
RandomCrop,
PadIfNeeded,
RandomScale,
LongestMaxSize,
SmallestMaxSize,
OneOf,
)
from albumentations.pytorch import ToTensorV2 as ToTensor
from densetorch.data import albumentations2densetorch
if dataset_type == "densetorch":
wrapper = albumentations2densetorch
elif dataset_type == "torchvision":
wrapper = albumentations2torchvision
else:
raise ValueError(f"Unknown dataset type: {dataset_type}")
common_transformations = [
Normalize(max_pixel_value=1.0 / img_scale, mean=img_mean, std=img_std),
ToTensor(),
]
train_transforms = []
for stage in range(num_stages):
train_transforms.append(
wrapper([
ChangeBackground("../backgrounds", p=0.5), # my addition
MotionBlur(p=0.5),
OneOf([
RandomScale(scale_limit=(low_scale[stage],
high_scale[stage])),
LongestMaxSize(max_size=shorter_side[stage]),
SmallestMaxSize(max_size=shorter_side[stage]),
]),
PadIfNeeded(
min_height=crop_size[stage],
min_width=crop_size[stage],
border_mode=cv2.BORDER_CONSTANT,
value=np.array(img_mean) / img_scale,
mask_value=ignore_label,
),
HorizontalFlip(p=0.5, ),
RandomRotate90(p=0.5),
RandomBrightnessContrast(
p=.8), # only applies to images, not masks
RandomGamma(p=0.8), # only applies to images
OneOf(
[
ElasticTransform(p=0.5,
alpha=120,
sigma=500 * 0.05,
alpha_affine=500 * 0.03),
GridDistortion(p=0.5),
# A.OpticalDistortion(distort_limit=1, shift_limit=0.5, p=1),
],
p=.5),
RandomCrop(
height=crop_size[stage],
width=crop_size[stage],
),
] + common_transformations))
val_transforms = wrapper(common_transformations)
return train_transforms, val_transforms
def get_data_transform(args):
#print(args)
# read general parameters
key = "GeneralParams"
if key not in args:
print("Mandatory {} attribute is missing".format(key))
return None, None
inp_size = args[key]["input_size"]
# prepare Augmentation: Normalized
key = "Normalize"
if key not in args:
print("Mandatory {} attribute is missing".format(key))
return None, None
normalize = albumentations.Normalize(mean=args[key]["means"], std=args[key]["stds"])
resize = albumentations.Resize(inp_size,inp_size)
# prepare Augmentation: CoarseDropout (same as cutout)
cutout = None
key = "CoarseDropout"
if key in args and args[key]["apply"]:
print("{}/Cutout is enabled".format(key))
cutout = albumentations.CoarseDropout(
max_holes=args[key]["max_holes"],
max_height=args[key]["max_height"], max_width=args[key]["max_width"],
min_height=args[key]["min_height"], min_width=args[key]["min_width"],
fill_value=args[key]["fill_value"],
p=args[key]["p"])
# prepare Augmentation: ElasticTransform
elasticTransform = None
key = "ElasticTransform"
if key in args and args[key]["apply"]:
print("{} is enabled".format(key))
elasticTransform = albumentations.ElasticTransform(
sigma=args[key]["sigma"],
alpha=args[key]["alpha"],
alpha_affine=args[key]["alpha_affine"],
p=args[key]["p"])
# prepare Augmentation: HorizontalFlip
horizontalFlip = None
key = "HorizontalFlip"
if key in args and args[key]["apply"]:
print("{} is enabled".format(key))
horizontalFlip = albumentations.HorizontalFlip(p=args[key]["p"])
# prepare Augmentation: RandomCrop
randomCrop = None
key = "RandomCrop"
if key in args and args[key]["apply"]:
print("{} is enabled".format(key))
size = args[key]["size"]
randomCrop = albumentations.RandomCrop(size, size,
p=args[key]["p"])
# prepare train transform list
train_transform_list = []
# arrange all the transform in required order
if randomCrop is not None:
train_transform_list.append(randomCrop)
train_transform_list.append(resize)
if horizontalFlip is not None:
train_transform_list.append(horizontalFlip)
if elasticTransform is not None:
train_transform_list.append(elasticTransform)
if cutout is not None:
train_transform_list.append(cutout)
train_transform_list.append(normalize)
train_transform_list.append(ToTensor())
train_transforms = albumentations.Compose(train_transform_list)
train_transforms = AlbumCompose(train_transforms)
# Test Phase transformations
test_transforms = albumentations.Compose([
#resize,
normalize,
ToTensor()
])
test_transforms = AlbumCompose(test_transforms)
return train_transforms, test_transforms
def __init__(self):
self.Compose = albu.Compose([
albu.HorizontalFlip(p=0),
albu.ToFloat(max_value=None, always_apply=True),
ToTensor(normalize={'mean': [0.5071, 0.4867, 0.4408], 'std': [0.2675, 0.2565, 0.2761]})
])
def __init__(self):
self.transform = Compose(
[
Normalize((0.4914, 0.4822, 0.4465), ((0.2023, 0.1994, 0.2010))),
ToTensor(),
])
def compose_augmentations(img_height,
img_width,
flip_p=0.5,
translate_p=0.5,
distort_p=0.5,
color_p=0.5,
overlays_p=0.15,
blur_p=0.25,
noise_p=0.25):
# Resize
resize_p = 1 if img_height != 1024 else 0
# Random sized crop
if img_height == 1024:
min_max_height = (896, 960)
elif img_height == 512:
min_max_height = (448, 480)
elif img_height == 256:
min_max_height = (224, 240)
else:
raise NotImplementedError
return Compose([
Resize(p=resize_p, height=img_height, width=img_width),
OneOf([
HorizontalFlip(p=0.5),
VerticalFlip(p=0.5),
Transpose(p=0.5),
RandomRotate90(p=0.5),
],
p=flip_p),
OneOf([
Rotate(p=0.25, limit=10),
RandomSizedCrop(p=0.5,
min_max_height=min_max_height,
height=img_height,
width=img_width),
OneOrOther(IAAAffine(p=0.1, translate_percent=0.05),
IAAPerspective(p=0.1)),
],
p=translate_p),
OneOf([
ElasticTransform(p=0.5,
alpha=10,
sigma=img_height * 0.05,
alpha_affine=img_height * 0.03,
approximate=True),
GridDistortion(p=0.5),
OpticalDistortion(p=0.5),
IAAPiecewiseAffine(p=0.25, scale=(0.01, 0.03)),
],
p=distort_p),
OneOrOther(
OneOf([
CLAHE(p=0.5),
RandomGamma(p=0.5),
RandomContrast(p=0.5),
RandomBrightness(p=0.5),
RandomBrightnessContrast(p=0.5),
],
p=color_p),
OneOf([IAAEmboss(p=0.1),
IAASharpen(p=0.1),
IAASuperpixels(p=0)],
p=overlays_p)),
OneOrOther(
OneOf([
Blur(p=0.2),
MedianBlur(p=0.1),
MotionBlur(p=0.1),
GaussianBlur(p=0.1),
],
p=blur_p),
OneOf([GaussNoise(p=0.2),
IAAAdditiveGaussianNoise(p=0.1)],
p=noise_p)),
Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
ToTensor(sigmoid=False),
])
def post_transforms():
return albu.Compose([albu.Normalize(), ToTensor()],
p=1,
bbox_params=albu.BboxParams(format='pascal_voc'))
def __init__(self,
resize=(0, 0),
padding=(0, 0),
crop=(0, 0),
horizontal_flip_prob=0.0,
vertical_flip_prob=0.0,
gaussian_blur_prob=0.0,
rotate_degree=0.0,
cutout_prob=0.0,
cutout_dim=(8, 8),
hue_saturation_prob=0.0,
contrast_prob=0.0,
mean=(0.5, 0.5, 0.5),
std=(0.5, 0.5, 0.5),
normalize=True,
train=True):
"""Create data transformation pipeline.
Args:
resize (tuple, optional): Resize the input to the given height and
width. (default: (0, 0))
padding (tuple, optional): Pad the image if the image size is less
than the specified dimensions (height, width). (default= (0, 0))
crop (tuple, optional): Randomly crop the image with the specified
dimensions (height, width). (default: (0, 0))
horizontal_flip_prob (float, optional): Probability of an image
being horizontally flipped. (default: 0)
vertical_flip_prob (float, optional): Probability of an image
being vertically flipped. (default: 0)
rotate_prob (float, optional): Probability of an image being
rotated. (default: 0)
rotate_degree (float, optional): Angle of rotation for image
augmentation. (default: 0)
cutout_prob (float, optional): Probability that cutout will be
performed. (default: 0)
cutout_dim (tuple, optional): Dimensions of the cutout box (height, width).
(default: (8, 8))
hue_saturation_prob (float, optional): Probability of randomly changing hue,
saturation and value of the input image. (default: 0)
contrast_prob (float, optional): Randomly changing contrast of the input image.
(default: 0)
mean (float or tuple, optional): Dataset mean. (default: 0.5 for each channel)
std (float or tuple, optional): Dataset standard deviation. (default: 0.5 for each channel)
"""
transforms_list = []
if sum(resize) > 0:
transforms_list += [
A.Resize(height=resize[0], width=resize[1], always_apply=True)
]
if train:
if sum(padding) > 0:
transforms_list += [
A.PadIfNeeded(min_height=padding[0],
min_width=padding[1],
always_apply=True)
]
if sum(crop) > 0:
transforms_list += [
A.RandomCrop(crop[0], crop[1], always_apply=True)
]
if horizontal_flip_prob > 0: # Horizontal Flip
transforms_list += [A.HorizontalFlip(p=horizontal_flip_prob)]
if vertical_flip_prob > 0: # Vertical Flip
transforms_list += [A.VerticalFlip(p=vertical_flip_prob)]
if gaussian_blur_prob > 0: # Patch Gaussian Augmentation
transforms_list += [A.GaussianBlur(p=gaussian_blur_prob)]
if rotate_degree > 0: # Rotate image
transforms_list += [A.Rotate(limit=rotate_degree)]
if cutout_prob > 0: # CutOut
if isinstance(mean, float):
fill_value = mean * 255.0
else:
fill_value = tuple([x * 255.0 for x in mean])
transforms_list += [
A.CoarseDropout(p=cutout_prob,
max_holes=1,
fill_value=fill_value,
max_height=cutout_dim[0],
max_width=cutout_dim[1])
]
if hue_saturation_prob > 0: # Hue Saturation
transforms_list += [
A.HueSaturationValue(p=hue_saturation_prob)
]
if contrast_prob > 0: # Random Contrast
transforms_list += [A.RandomContrast(p=contrast_prob)]
if normalize:
# normalize the data with mean and standard deviation to keep values in range [-1, 1]
# since there are 3 channels for each image,
# we have to specify mean and std for each channel
transforms_list += [
A.Normalize(mean=mean, std=std, always_apply=True),
]
# convert the data to torch.FloatTensor
transforms_list += [ToTensor()]
self.transform = A.Compose(transforms_list)
mean = (0.4914, 0.4822, 0.4465)
std = (0.2023, 0.1994, 0.2010)
mean_array = np.array([*mean])
train_transforms = Compose([
PadIfNeeded(40, 40, always_apply=True, p=1.0),
RandomCrop(32, 32, always_apply=True, p=1.0),
HorizontalFlip(p=0.5),
Cutout(num_holes=1,
max_h_size=8,
max_w_size=8,
fill_value=np.array([*mean]) * 255.0,
p=0.75),
Normalize(mean, std),
ToTensor()
])
test_transforms = Compose([Normalize(mean, std), ToTensor()])
data = DataUtility.download_CIFAR10(Alb(train_transforms),
Alb(test_transforms),
batch_size=512)
net = S11Resnet().to(Utility.getDevice())
optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)
builder = ModelBuilder(net, data, LossFn(F.nll_loss, l2Factor=0.01, model=net),
optimizer)
result = builder.fit(1)
def main():
device = torch.device("cuda" if not hyperparams.
hyperparameter_defaults['no_cuda'] else "cpu")
hyperparams.hyperparameter_defaults['run_name'] = fileutils.rand_run_name()
transform_train = Compose([
PadIfNeeded(min_height=40, min_width=40, always_apply=True,
p=1.0), #,value=(0,0,0), border_mode=0),
RandomCrop(height=32, width=32, p=1),
#Flip(p=0.5),
IAAFliplr(p=1),
Cutout(num_holes=1,
max_h_size=8,
max_w_size=8,
always_apply=True,
p=1,
fill_value=[0.4914 * 255, 0.4826 * 255, 0.44653 * 255]),
Normalize(
mean=[0.4914, 0.4826, 0.44653],
std=[0.24703, 0.24349, 0.26519],
),
ToTensor()
])
trainloader, testloader = dataloader.get_train_test_dataloader_cifar10(
transform_train=transform_train)
print("Initializing datasets and dataloaders")
#model_new = basemodelclass.ResNet18(hyperparams.hyperparameter_defaults['dropout'])
model_new = basemodelclass.S11ResNet()
wandb_run_init = wandb.init(
config=hyperparams.hyperparameter_defaults,
project=hyperparams.hyperparameter_defaults['project'])
wandb.watch_called = False
config = wandb.config
print(config)
wandb.watch(model_new, log="all")
#trainloader, testloader = dataloader.get_train_test_dataloader_cifar10()
optimizer = optim.SGD(model_new.parameters(),
lr=config.lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
#optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
criterion = nn.CrossEntropyLoss
#scheduler = None
#scheduler = CyclicLR(optimizer, base_lr=config.lr*0.01, max_lr=config.lr, mode='triangular', gamma=1., cycle_momentum=True,step_size_up=2000)#, scale_fn='triangular',step_size_up=200)
scheduler = OneCycleLR(optimizer,
config.ocp_max_lr,
epochs=config.epochs,
cycle_momentum=False,
steps_per_epoch=len(trainloader),
base_momentum=config.momentum,
max_momentum=0.95,
pct_start=0.208,
anneal_strategy=config.anneal_strategy,
div_factor=config.div_factor,
final_div_factor=config.final_div_factor)
#scheduler =CyclicLR(optimizer, base_lr=config.lr*0.01, max_lr=config.lr, mode='triangular', gamma=1., cycle_momentum=True,step_size_up=2000)#, scale_fn='triangular',step_size_up=200)
#scheduler = StepLR(optimizer, step_size=config.sched_lr_step, gamma=config.sched_lr_gamma)
#scheduler = MultiStepLR(optimizer, milestones=[10,20], gamma=config.sched_lr_gamma)
#scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=2, verbose=True, threshold=0.0001)
#scheduler = traintest.MyOwnReduceLROnPlateau(optimizer, mode='min', factor=0.2, patience=2, verbose=True, threshold=0.0001)
#scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=config.factor, patience=4, verbose=True, threshold=config.lr_decay_threshold)
final_model_path = traintest.execute_model(
model_new,
hyperparams.hyperparameter_defaults,
trainloader,
testloader,
device,
dataloader.classes,
wandb=wandb,
optimizer_in=optimizer,
scheduler=scheduler,
prev_saved_model=saved_model_path,
criterion=criterion,
save_best=True,
lars_mode=False,
batch_step=True)
def __init__(self,
batch_size=64,
use_cuda=True,
dataset_name='cifar10',
trainset=None,
testset=None,
train_transforms=None,
test_transforms=None):
super().__init__()
self.dataset_name = dataset_name
if self.dataset_name == 'cifar10':
if trainset:
self.trainset = trainset
else:
# Train Phase transformations
if train_transforms:
self.train_transforms = train_transforms
else:
self.train_transforms = train_transforms = Compose([
Rotate(limit=15),
Cutout(num_holes=8,
max_h_size=12,
max_w_size=8,
always_apply=False,
p=0.5),
# RandomSizedCrop(min_max_height=(20, 32), height=32, width=32),
HorizontalFlip(),
Normalize(
mean=[0.4914, 0.4822, 0.4465],
std=[0.2471, 0.2435, 0.2616],
),
ToTensor()
])
self.trainset = CIFAR10_dataset(
root='./data',
train=True,
download=True,
transform=self.train_transforms)
if testset:
self.testset = testset
else:
# Test Phase transformations
if test_transforms:
self.test_transforms = test_transforms
else:
self.test_transforms = Compose([
Normalize(
mean=[0.4914, 0.4822, 0.4465],
std=[0.2471, 0.2435, 0.2616],
),
ToTensor()
])
self.testset = CIFAR10_dataset(root='./data',
train=False,
download=True,
transform=self.test_transforms)
# dataloader arguments - something you'll fetch these from cmdprmt
dataloader_args = dict(shuffle=True,
batch_size=batch_size,
num_workers=4,
pin_memory=True) if use_cuda else dict(
shuffle=True, batch_size=batch_size)
# train dataloader
self.train_loader = torch.utils.data.DataLoader(
self.trainset, **dataloader_args)
# test dataloader
self.test_loader = torch.utils.data.DataLoader(
self.testset, **dataloader_args)
def __init__(self):
self.trans=A.Compose(
[A.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),ToTensor()])
def post_transforms():
# we use ImageNet image normalization
# and convert it to torch.Tensor
return [albu.Normalize(), ToTensor()]
return {'image': img, 'labels': labels}
else:
return {'image': img}
# Data loaders
transform_train = Compose([
ShiftScaleRotate(),
HorizontalFlip(p=0.5, ),
VerticalFlip(p=0.5, ),
RandomBrightnessContrast(0.08, 0.08),
ToTensor()
])
transform_validation = Compose([ToTensor()])
transform_test = Compose([ToTensor()])
train = pd.read_csv(os.path.join(args.dir_csv, 'stage_1_train.csv'))
test = pd.read_csv(os.path.join(args.dir_csv, 'stage_1_sample_submission.csv'))
train[['ID', 'Image', 'Diagnosis']] = train['ID'].str.split('_', expand=True)
train = train[['Image', 'Diagnosis', 'Label']]
train.drop_duplicates(inplace=True)
train = train.pivot(index='Image', columns='Diagnosis',
values='Label').reset_index()
train['Image'] = 'ID_' + train['Image']
from albumentations import Compose, Resize, RandomCrop, Flip, HorizontalFlip, VerticalFlip, Transpose, RandomRotate90, \
ShiftScaleRotate, OneOf, Blur, MotionBlur, MedianBlur, GaussianBlur
from albumentations.pytorch import ToTensor
train_aug = Compose([
RandomCrop(height=96, width=96, p=0.4),
OneOf([
VerticalFlip(p=0.2),
HorizontalFlip(p=0.3),
Transpose(p=0.2),
RandomRotate90(p=0.2),
],
p=0.5),
ShiftScaleRotate(p=0.5),
OneOf([
Blur(p=0.2),
MotionBlur(p=0.2),
MedianBlur(p=0.2),
GaussianBlur(p=0.2),
],
p=0.4),
Resize(128, 128, always_apply=True),
ToTensor()
])
valid_aug = Compose([Resize(128, 128, always_apply=True), ToTensor()])
device = torch.device('cuda') #utils.py 안에 device는 따로 세팅해줘야함
with open('./dataset', 'rb') as file:
data = pickle.load(file)
# In[4]:
transformers = Compose([
OneOf(
[
# augs.VinylShining(1),
augs.GridMask(num_grid=(10, 20)),
augs.RandomAugMix(severity=2, width=2)
],
p=0.7),
ToTensor()
])
train_custom = utils.CustomDataset_jamo(data[:int(len(data) * 0.98)],
resize_shape=(opt.imgH, opt.imgW),
transformer=transformers)
valid_custom = utils.CustomDataset_jamo(data[int(len(data) * 0.98):],
resize_shape=(opt.imgH, opt.imgW),
transformer=ToTensor())
data_loader = DataLoader(train_custom,
batch_size=opt.batch_size,
num_workers=15,
shuffle=True,
drop_last=True,
pin_memory=True)
valid_loader = DataLoader(valid_custom,
