def predict():
save_path = 'output/predictions/nn_models/'
save_path1 = 'output/predictions/nn_models/test/'
if not os.path.exists(os.path.dirname(save_path1)):
os.makedirs(os.path.dirname(save_path1))
save_path2 = 'output/predictions/nn_models/valid/'
if not os.path.exists(os.path.dirname(save_path2)):
os.makedirs(os.path.dirname(save_path2))
mods = [str(50), str(101), str(152), str(161)]
path = pathlib.PosixPath('output/preprocessing')
classes = os.listdir('output/preprocessing/croped_images/')
bs = 50
size = 224
version = str(0)
tfms = fastai.vision.transform.get_transforms(do_flip=False,
flip_vert=False,
max_rotate=10,
max_zoom=1.1,
max_lighting=0.2,
p_affine=0.85,
p_lighting=0.4,
max_warp=0.2,
xtra_tfms=None)
data_test = (fastai.vision.ImageList.from_folder(
path).split_none().label_from_folder().transform(
tfms, size=size).add_test_folder().databunch(bs=bs).normalize(
fastai.vision.imagenet_stats))
with open(save_path + 'test_items.csv', 'w') as myfile:
wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
wr.writerow(data_test.test_ds.items)
im_res = {}
folds = 10
for mod in mods:
valid_items = []
for fold in range(folds):
train_df = pd.read_csv(
'output/preprocessing/folds/fold_{}.csv'.format(fold),
dtype=str)
val_df = pd.read_csv(
'output/preprocessing/folds/val_{}.csv'.format(fold),
dtype=str)
data = (fastai.vision.ImageList.from_df(
df=train_df, path=path / 'croped_images',
cols='name').split_none().label_from_df(
cols='label',
classes=classes).transform(tfms, size=size).databunch(
bs=bs).normalize(fastai.vision.imagenet_stats))
valid = (fastai.vision.ImageList.from_df(
df=val_df, path=path / 'croped_images',
cols='name').split_none().label_from_df(
cols='label',
classes=classes).transform(tfms, size=size).databunch(
bs=bs).normalize(fastai.vision.imagenet_stats))
data.valid_dl = valid.train_dl
if mod == '50':
learn = fastai.vision.learner.cnn_learner(
data=data,
base_arch=model_50,
cut=-2,
ps=0.2,
metrics=fastai.metrics.accuracy,
model_dir='../../training/models')
elif mod == '101':
learn = fastai.vision.learner.cnn_learner(
data=data,
base_arch=model_101,
cut=-2,
ps=0.2,
metrics=fastai.metrics.accuracy,
model_dir='../../training/models')
elif mod == '152':
learn = fastai.vision.learner.cnn_learner(
data=data,
base_arch=model_152,
cut=-2,
ps=0.2,
metrics=fastai.metrics.accuracy,
model_dir='../../training/models')
elif mod == '161':
learn = fastai.vision.learner.cnn_learner(
data=data,
base_arch=model_161,
cut=-1,
ps=0.2,
metrics=fastai.metrics.accuracy,
model_dir='../../training/models')
learn.model = nn.DataParallel(learn.model)
learn.load('model_' + mod + '_' + version + '_fold_' + str(fold))
learn.data = data_test
pr, y = learn.TTA(ds_type=fastai.basic_data.DatasetType.Test)
torch.save(
pr, save_path + 'test/test_' + mod + '_' + version + '_fold_' +
str(fold) + '.pt')
learn = None
data = None
gc.collect()
torch.cuda.empty_cache()
def train_50():
mod = str(50)
if not os.path.exists(os.path.dirname('output/training/models')):
os.makedirs(os.path.dirname('output/training/models'))
path = pathlib.PosixPath('output/preprocessing')
classes = os.listdir('output/preprocessing/croped_images/')
bs = 140
size = 224
version = str(0)
tfms = fastai.vision.transform.get_transforms(do_flip=False,
flip_vert=False, max_rotate=10,
max_zoom=1.1, max_lighting=0.2,
p_affine=0.85, p_lighting=0.4,
max_warp=0.2, xtra_tfms=None)
data_test = (fastai.vision.ImageList.from_folder(path)
.split_none()
.label_from_folder()
.transform(tfms, size=size)
.add_test_folder()
.databunch(bs=bs)
.normalize(fastai.vision.imagenet_stats))
im_res= {}
folds = 10
for fold in range(folds):
print(fold)
train_df = pd.read_csv('output/preprocessing/folds/fold_{}.csv'.format(fold), dtype=str)
val_df = pd.read_csv('output/preprocessing/folds/val_{}.csv'.format(fold), dtype=str)
data = (fastai.vision.ImageList.from_df(df=train_df, path=path/'croped_images', cols='name')
.split_none()
.label_from_df(cols='label', classes=classes)
.transform(tfms, size=size)
.databunch(bs=bs)
.normalize(fastai.vision.imagenet_stats))
valid = (fastai.vision.ImageList.from_df(df=val_df, path=path/'croped_images', cols='name')
.split_none()
.label_from_df(cols='label', classes=classes)
.transform(tfms, size=size)
.databunch(bs=bs)
.normalize(fastai.vision.imagenet_stats))
data.valid_dl = valid.train_dl
learn = fastai.vision.learner.cnn_learner(
data = data,
base_arch=model_50,
cut=-2,
ps=0.2,
metrics=fastai.metrics.accuracy,
model_dir='../../training/models')
learn.loss_fn = FocalLoss()
learn.crit = FocalLoss()
learn.model = nn.DataParallel(learn.model)
learn.fit_one_cycle(3, max_lr=3e-3) #3
write_log(fold, version, learn, mod)
learn.unfreeze()
lr = 9e-4
lrs=np.array([lr/10, lr/3])
learn.fit_one_cycle(4, lrs)
write_log(fold, version, learn, mod)
# lr = 5e-4
# lrs=np.array([lr/10, lr/3])
# learn.fit_one_cycle(4, lrs)
# write_log(fold, version, learn, mod)
learn.freeze()
learn.save('model_'+mod+'_'+version+'_fold_'+str(fold))
learn = None
data = None
gc.collect()
torch.cuda.empty_cache()