def train(optimizer, train_optimizer=train_optimizer):
# load and crop validation data
print "Loading data ..."
X_val = np.load(os.path.join(args.data_path, 'X_val.npy'))
y_val = np.load(os.path.join(args.data_path, 'y_val.npy'))
manip_val = np.zeros(
(len(y_val), 1), dtype=np.float32
) # np.load(os.path.join(args.data_path, 'manip_with_pseudo.npy')) # 68/480 manipulated
c = args.crop_size
C = X_val.shape[1]
if c < C:
X_val = X_val[:, C / 2 - c / 2:C / 2 + c / 2,
C / 2 - c / 2:C / 2 + c / 2, :]
if args.kernel:
X_val = [conv_K(x) for x in X_val]
# make validation loader
rng = RNG(args.random_seed + 42 if args.random_seed else None)
val_transform = transforms.Compose([
transforms.Lambda(lambda (x, m, y): (Image.fromarray(x), m, y)),
########
# 1 - (480-68-0.3*480)/(480-68) ~ 0.18
########
transforms.Lambda(lambda (img, m, y): (make_random_manipulation(img, rng, crop_policy='center'), float32(1.), y) if\
m[0] < 0.5 and rng.rand() < VAL_MANIP_RATIO else (img, m, y)),
transforms.Lambda(lambda (img, m, y): ([img,
img.transpose(Image.ROTATE_90)][int(rng.rand() < 0.5)], m) if \
KaggleCameraDataset.is_rotation_allowed()[y] else (img, m)),
transforms.Lambda(lambda (img, m): (transforms.ToTensor()(img), m)),
transforms.Lambda(lambda (img, m): (transforms.Normalize(args.means, args.stds)(img), m))
])
np.save(os.path.join(args.model_dirpath, 'y_val.npy'), np.vstack(y_val))
val_dataset = make_numpy_dataset(X=[
(x, m, y) for x, m, y in zip(X_val, manip_val, y_val)
],
y=y_val,
transform=val_transform)
val_loader = DataLoader(dataset=val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_workers)
n_runs = args.epochs / args.epochs_per_unique_data + 1
for _ in xrange(n_runs):
train_loader = make_train_loaders(block_index=optimizer.epoch /
args.epochs_per_unique_data)
optimizer.max_epoch = optimizer.epoch + args.epochs_per_unique_data
train_optimizer(optimizer, train_loader, val_loader)
def _make_predict_train_loader(X_b, manip_b, manip_ratio=0.):
assert len(X_b) == len(manip_b)
# make dataset
rng = RNG(1337)
train_transforms_list = [
transforms.Lambda(lambda (x, m): (Image.fromarray(x), m)),
# if `val` == False
# 972/1982 manip pseudo images
# images : pseudo = approx. 48 : 8 = 6 : 1
# to get unalt : manip = 70 : 30 (like in test metric),
# we manip ~24.7% of non-pseudo images
# else:
# we simply use same ratio as in validation (0.18)
transforms.Lambda(lambda (img, m): (make_random_manipulation(img, rng, crop_policy='center', crop_size=512), float32(1.)) if \
m[0] < 0.5 and rng.rand() < manip_ratio else (center_crop(img, 512), m))
]
train_transforms_list += make_aug_transforms(rng)
if args.crop_size == 512:
train_transforms_list += [
transforms.Lambda(lambda (
img, m): ([img, img.transpose(Image.ROTATE_90)], [m] * 2)),
transforms.Lambda(lambda (crops, ms): (torch.stack([
transforms.Normalize(args.means, args.stds)
(transforms.ToTensor()(crop)) for crop in crops
]), torch.from_numpy(np.asarray(ms))))
]
else:
train_transforms_list += [
transforms.Lambda(lambda (img, m): (transforms.TenCrop(
args.crop_size)(img), [m] * 10)),
transforms.Lambda(lambda (imgs, ms): (list(
imgs) + [img.transpose(Image.ROTATE_90)
for img in imgs], ms + ms)),
transforms.Lambda(lambda (crops, ms): (torch.stack([
transforms.Normalize(args.means, args.stds)
(transforms.ToTensor()(crop)) for crop in crops
]), torch.from_numpy(np.asarray(ms))))
]
train_transform = transforms.Compose(train_transforms_list)
dataset = make_numpy_dataset(X=[(x, m) for x, m in zip(X_b, manip_b)],
y=np.zeros(len(X_b), dtype=np.int64),
transform=train_transform)
# make loader
loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_workers)
return loader
def make_train_loaders(block_index):
# assemble data
X_train = []
y_train = []
manip_train = []
for c in xrange(10):
X_block = np.load(
os.path.join(args.data_path,
'X_{0}_{1}.npy'.format(c, block_index % N_BLOCKS[c])))
X_block = [X_block[i] for i in xrange(len(X_block))]
if args.bootstrap:
X_block = [X_block[i] for i in b_ind[c][block_index % N_BLOCKS[c]]]
X_train += X_block
y_train += np.repeat(c, len(X_block)).tolist()
manip_train += [float32(0.)] * len(X_block)
for c in xrange(10):
X_pseudo_block = np.load(
os.path.join(
args.data_path, 'X_pseudo_{0}_{1}.npy'.format(
c, block_index % N_PSEUDO_BLOCKS[c])))
X_pseudo_block = [
X_pseudo_block[i] for i in xrange(len(X_pseudo_block))
]
if args.bootstrap:
X_pseudo_block = [
X_pseudo_block[i]
for i in b_pseudo_ind[c][block_index % N_PSEUDO_BLOCKS[c]]
]
X_train += X_pseudo_block
y_train += np.repeat(c, len(X_pseudo_block)).tolist()
manip_block = np.load(
os.path.join(
args.data_path, 'manip_pseudo_{0}_{1}.npy'.format(
c, block_index % N_PSEUDO_BLOCKS[c])))
manip_block = [m for m in manip_block]
if args.bootstrap:
manip_block = [
manip_block[i]
for i in b_pseudo_ind[c][block_index % N_PSEUDO_BLOCKS[c]]
]
manip_train += manip_block
shuffle_ind = range(len(y_train))
RNG(seed=block_index).shuffle(shuffle_ind)
X_train = [X_train[i] for i in shuffle_ind]
y_train = [y_train[i] for i in shuffle_ind]
manip_train = [manip_train[i] for i in shuffle_ind]
# make dataset
rng = RNG(args.random_seed)
train_transforms_list = [
transforms.Lambda(lambda (x, m, y): (Image.fromarray(x), m, y)),
######
# 972/1982 manip pseudo images
# images : pseudo = approx. 48 : 8 = 6 : 1
# thus to get 50 : 50 manip : unalt we manip 11965/25874 ~ 46% of non-pseudo images
######
transforms.Lambda(lambda (img, m, y): (make_random_manipulation(img, rng), float32(1.), y) if \
m[0] < 0.5 and rng.rand() < TRAIN_MANIP_RATIO else (make_crop(img, args.crop_size, rng), m, y)),
transforms.Lambda(lambda (img, m, y): ([img,
img.transpose(Image.ROTATE_90)][int(rng.rand() < 0.5)], m) if \
KaggleCameraDataset.is_rotation_allowed()[y] else (img, m)),
]
train_transforms_list += make_aug_transforms(rng)
if args.kernel:
train_transforms_list += [
transforms.Lambda(lambda (img, m):
(conv_K(np.asarray(img, dtype=np.uint8)), m)),
transforms.Lambda(lambda (x, m):
(torch.from_numpy(x.transpose(2, 0, 1)), m))
]
else:
train_transforms_list += [
transforms.Lambda(lambda (img, m): (transforms.ToTensor()(img), m))
]
train_transforms_list += [
transforms.Lambda(lambda (img, m): (transforms.Normalize(
args.means, args.stds)(img), m))
]
train_transform = transforms.Compose(train_transforms_list)
dataset = make_numpy_dataset(X=[
(x, m, y) for x, m, y in zip(X_train, manip_train, y_train)
],
y=y_train,
transform=train_transform)
# make loader
loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_workers,
sampler=StratifiedSampler(
class_vector=np.asarray(y_train),
batch_size=args.batch_size))
return loader
def predict(optimizer, **kwargs):
# load data
X_test = np.load(os.path.join(kwargs['data_path'], 'X_test.npy'))
y_test = np.zeros((len(X_test), ), dtype=np.int64)
test_transform = transforms.Compose([
transforms.Lambda(lambda x: Image.fromarray(x)),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
# TTA
rng = RNG(seed=1337)
base_transform = transforms.Compose([
transforms.Lambda(lambda x: Image.fromarray(x)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.Lambda(
lambda img: [img, img.transpose(Image.ROTATE_90)][int(rng.rand() <
0.5)]),
transforms.Lambda(
lambda img: adjust_gamma(img, gamma=rng.uniform(0.8, 1.25))),
transforms.Lambda(
lambda img: jpg_compress(img, quality=rng.randint(70, 100 + 1))),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
tta_n = 10
def tta_f(img, n=tta_n - 1):
out = [test_transform(img)]
for _ in xrange(n):
out.append(base_transform(img))
return torch.stack(out, 0)
tta_transform = transforms.Compose([
transforms.Lambda(lambda img: tta_f(img)),
])
test_loader = DataLoader(dataset=make_numpy_dataset(
X_test, y_test, tta_transform),
batch_size=kwargs['batch_size'],
shuffle=False,
num_workers=4)
test_dataset = KaggleCameraDataset(kwargs['data_path'],
train=False,
lazy=not kwargs['not_lazy'])
# compute predictions
logits, _ = optimizer.test(test_loader)
# compute and save raw probs
logits = np.vstack(logits)
proba = softmax(logits)
# group and average predictions
K = 16 * tta_n
proba = proba.reshape(len(proba) / K, K, -1).mean(axis=1)
fnames = [os.path.split(fname)[-1] for fname in test_dataset.X]
df = pd.DataFrame(proba)
df['fname'] = fnames
df = df[['fname'] + range(10)]
dirpath = os.path.split(kwargs['predict_from'])[0]
df.to_csv(os.path.join(dirpath, 'proba.csv'), index=False)
# compute predictions and save in submission format
index_pred = unhot(one_hot_decision_function(proba))
data = {
'fname': fnames,
'camera':
[KaggleCameraDataset.target_labels()[int(c)] for c in index_pred]
}
df2 = pd.DataFrame(data, columns=['fname', 'camera'])
df2.to_csv(os.path.join(dirpath, 'submission.csv'), index=False)
def train(optimizer, **kwargs):
# load training data
print 'Loading and splitting data ...'
if os.path.isfile(os.path.join(kwargs['data_path'], 'X_train.npy')):
X_train = np.load(os.path.join(kwargs['data_path'], 'X_train.npy'))
y_train = np.load(os.path.join(kwargs['data_path'], 'y_train.npy'))
X_val = np.load(os.path.join(kwargs['data_path'], 'X_val.npy'))
y_val = np.load(os.path.join(kwargs['data_path'], 'y_val.npy'))
else:
X = np.load(os.path.join(kwargs['data_path'], 'X_patches.npy'))
y = np.load(os.path.join(kwargs['data_path'], 'y_patches.npy'))
# split into train, val in stratified fashion
sss = StratifiedShuffleSplit(n_splits=1,
test_size=kwargs['n_val'],
random_state=kwargs['random_seed'])
train_ind, val_ind = list(sss.split(np.zeros_like(y), y))[0]
X_train = X[train_ind]
y_train = y[train_ind]
X_val = X[val_ind]
y_val = y[val_ind]
np.save(os.path.join(kwargs['data_path'], 'X_train.npy'), X_train)
np.save(os.path.join(kwargs['data_path'], 'y_train.npy'), y_train)
np.save(os.path.join(kwargs['data_path'], 'X_val.npy'), X_val)
np.save(os.path.join(kwargs['data_path'], 'y_val.npy'), y_val)
rng = RNG()
# noinspection PyTypeChecker
train_transform = transforms.Compose([
transforms.Lambda(lambda x: Image.fromarray(x)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.Lambda(
lambda img: [img, img.transpose(Image.ROTATE_90)][int(rng.rand() <
0.5)]),
transforms.Lambda(
lambda img: adjust_gamma(img, gamma=rng.uniform(0.8, 1.25))),
transforms.Lambda(
lambda img: jpg_compress(img, quality=rng.randint(70, 100 + 1))),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
val_transform = transforms.Compose([
transforms.Lambda(lambda x: Image.fromarray(x)),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
train_dataset = make_numpy_dataset(X_train, y_train, train_transform)
val_dataset = make_numpy_dataset(X_val, y_val, val_transform)
# define loaders
train_loader = DataLoader(dataset=train_dataset,
batch_size=kwargs['batch_size'],
shuffle=False,
num_workers=4,
sampler=StratifiedSampler(
class_vector=y_train,
batch_size=kwargs['batch_size']))
val_loader = DataLoader(dataset=val_dataset,
batch_size=kwargs['batch_size'],
shuffle=False,
num_workers=4)
print 'Starting training ...'
optimizer.train(train_loader, val_loader)
def train(optimizer, train_optimizer=train_optimizer):
# load and crop validation data
print "Loading data ..."
X_val = np.load(os.path.join(args.data_path, 'X_val.npy'))
y_val = np.load(os.path.join(args.data_path, 'y_val.npy')).tolist()
manip_val = np.zeros(
(len(y_val), 1), dtype=np.float32
) # np.load(os.path.join(args.data_path, 'manip_with_pseudo.npy')) # 68/480 manipulated
d = args.crop_size * 2
D = X_val.shape[1]
if d < D:
X_val = X_val[:, D / 2 - d / 2:D / 2 + d / 2,
D / 2 - d / 2:D / 2 + d / 2, :]
if args.kernel:
X_val = [conv_K(x) for x in X_val]
X_val = [X_val[i] for i in xrange(len(X_val))]
manip_val = [manip_val[i] for i in xrange(len(manip_val))]
for b in xrange(N_PSEUDO_BLOCKS_FOR_VALIDATION):
for c in xrange(10):
X_block = np.load(
os.path.join(args.data_path,
'X_pseudo_{0}_{1}.npy'.format(c, b)))
y_val += [c] * len(X_block)
d = args.crop_size * 2
D = X_block.shape[1]
if d < D:
X_block = X_block[:, D / 2 - d / 2:D / 2 + d / 2,
D / 2 - d / 2:D / 2 + d / 2, :]
X_val += [X_block[i] for i in xrange(len(X_block))]
manip_block = np.load(
os.path.join(args.data_path,
'manip_pseudo_{0}_{1}.npy'.format(c, b)))
manip_val += [m for m in manip_block]
# make validation loader
rng = RNG(args.random_seed + 42 if args.random_seed else None)
val_transform = transforms.Compose([
transforms.Lambda(lambda (x, m, y): (Image.fromarray(x), m, y)),
########
# 1 - (480-68-0.3*480)/(480-68) ~ 0.18
########
transforms.Lambda(lambda (img, m, y): (make_random_manipulation(img, rng, crop_policy='center'), float32(1.), y) if\
m[0] < 0.5 and rng.rand() < VAL_MANIP_RATIO else (center_crop(img, args.crop_size), m, y)),
# transforms.Lambda(lambda (img, m, y): ([img,
# img.transpose(Image.ROTATE_90)][int(rng.rand() < 0.5)], m) if \
# True else (img, m)),
transforms.Lambda(lambda (img, m, y): (transforms.ToTensor()(img), m)),
transforms.Lambda(lambda (img, m): (transforms.Normalize(args.means, args.stds)(img), m))
])
np.save(os.path.join(args.model_dirpath, 'y_val.npy'), np.vstack(y_val))
val_dataset = make_numpy_dataset(X=[
(x, m, y) for x, m, y in zip(X_val, manip_val, y_val)
],
y=y_val,
transform=val_transform)
val_loader = DataLoader(dataset=val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_workers)
for _ in xrange(args.distill_epochs / args.epochs_per_unique_data):
train_loader = make_train_loaders(block_index=optimizer.epoch /
args.epochs_per_unique_data,
distill=True)
optimizer.max_epoch = optimizer.epoch + args.epochs_per_unique_data
train_optimizer(optimizer, train_loader, val_loader)
n_runs = args.epochs / args.epochs_per_unique_data + 1
for _ in xrange(n_runs):
optimizer.distill_cost = 0.
train_loader = make_train_loaders(block_index=optimizer.epoch /
args.epochs_per_unique_data,
distill=False)
optimizer.max_epoch = optimizer.epoch + args.epochs_per_unique_data
train_optimizer(optimizer, train_loader, val_loader)
def make_train_loaders(block_index, distill=True):
# assemble data
X_train = []
y_train = []
manip_train = []
soft_logits = []
if distill:
soft_logits_ind = []
for c in xrange(10):
b = block_index % N_BLOCKS[c]
X_block = np.load(
os.path.join(args.data_path, 'X_{0}_{1}.npy'.format(c, b)))
X_block = [X_block[i] for i in xrange(len(X_block))]
if args.bootstrap:
X_block = [X_block[i] for i in b_ind[c][b]]
X_train += X_block
y_train += np.repeat(c, len(X_block)).tolist()
manip_train += [float32(0.)] * len(X_block)
soft_logits_ind_block = SOFT_LOGITS_IND[c][b]
if args.bootstrap:
soft_logits_ind_block = [
soft_logits_ind_block[i] for i in b_ind[c][b]
]
soft_logits_ind += soft_logits_ind_block
for c in xrange(10):
b = N_PSEUDO_BLOCKS_FOR_VALIDATION + block_index % N_PSEUDO_BLOCKS[
c]
X_pseudo_block = np.load(
os.path.join(args.data_path,
'X_pseudo_{0}_{1}.npy'.format(c, b)))
X_pseudo_block = [
X_pseudo_block[i] for i in xrange(len(X_pseudo_block))
]
if args.bootstrap:
X_pseudo_block = [
X_pseudo_block[i] for i in b_pseudo_ind[c][b]
]
X_train += X_pseudo_block
y_train += np.repeat(c, len(X_pseudo_block)).tolist()
manip_block = np.load(
os.path.join(args.data_path,
'manip_pseudo_{0}_{1}.npy'.format(c, b)))
manip_block = [m for m in manip_block]
if args.bootstrap:
manip_block = [manip_block[i] for i in b_pseudo_ind[c][b]]
manip_train += manip_block
soft_logits_ind_block = SOFT_LOGITS_IND[10 + c][b]
if args.bootstrap:
soft_logits_ind_block = [
soft_logits_ind_block[i] for i in b_pseudo_ind[c][b]
]
soft_logits_ind += soft_logits_ind_block
soft_logits = np.load(os.path.join(
args.data_path, 'logits_train.npy')).astype(np.float32)
soft_logits -= soft_logits.mean(axis=1)[:, np.newaxis]
soft_logits = [
soft_logits[i] / max(args.temperature, 1.) for i in soft_logits_ind
]
else:
for c in xrange(10):
X_block = np.load(
os.path.join(args.data_path, G[c][block_index % len(G[c])]))
X_block = [X_block[i] for i in xrange(len(X_block))]
X_train += X_block
y_train += np.repeat(c, len(X_block)).tolist()
manip_train += [float32(0.)] * len(X_block)
soft_logits += [np.zeros((10), dtype=np.float32)] * len(X_block)
shuffle_ind = range(len(y_train))
RNG(seed=block_index * 41).shuffle(shuffle_ind)
X_train = [X_train[i] for i in shuffle_ind]
y_train = [y_train[i] for i in shuffle_ind]
manip_train = [manip_train[i] for i in shuffle_ind]
soft_logits = [soft_logits[i] for i in shuffle_ind]
# make dataset
rng = RNG(args.random_seed)
train_transforms_list = [
transforms.Lambda(lambda (x, m, y): (Image.fromarray(x), m, y)),
######
# 972/1982 manip pseudo images
# images : pseudo = approx. 48 : 8 = 6 : 1
# thus to get 50 : 50 manip : unalt we manip 11965/25874 ~ 46% of non-pseudo images
######
transforms.Lambda(lambda (img, m, y): (make_random_manipulation(img, rng), float32(1.), y) if \
m[0] < 0.5 and rng.rand() < TRAIN_MANIP_RATIO else (make_crop(img, args.crop_size, rng), m, y)),
transforms.Lambda(lambda (img, m, y): ([img,
img.transpose(Image.ROTATE_90)][int(rng.rand() < 0.5)], m) if \
True else (img, m)),
]
train_transforms_list += make_aug_transforms(rng)
if args.kernel:
train_transforms_list += [
transforms.Lambda(lambda (img, m):
(conv_K(np.asarray(img, dtype=np.uint8)), m)),
transforms.Lambda(lambda (x, m):
(torch.from_numpy(x.transpose(2, 0, 1)), m))
]
else:
train_transforms_list += [
transforms.Lambda(lambda (img, m): (transforms.ToTensor()(img), m))
]
train_transforms_list += [
transforms.Lambda(lambda (img, m): (transforms.Normalize(
args.means, args.stds)(img), m))
]
train_transform = transforms.Compose(train_transforms_list)
dataset = make_numpy_dataset(X=[
(x, m, y) for x, m, y in zip(X_train, manip_train, y_train)
],
y=y_train,
transform=train_transform,
soft_logits=soft_logits)
# make loader
loader = DataLoader(dataset=dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.n_workers,
sampler=StratifiedSampler(
class_vector=np.asarray(y_train),
batch_size=args.batch_size))
return loader