def predict(): save_path = 'output/predictions/nn_models/' save_path1 = 'output/predictions/nn_models/test/' if not os.path.exists(os.path.dirname(save_path1)): os.makedirs(os.path.dirname(save_path1)) save_path2 = 'output/predictions/nn_models/valid/' if not os.path.exists(os.path.dirname(save_path2)): os.makedirs(os.path.dirname(save_path2)) mods = [str(50), str(101), str(152), str(161)] path = pathlib.PosixPath('output/preprocessing') classes = os.listdir('output/preprocessing/croped_images/') bs = 50 size = 224 version = str(0) tfms = fastai.vision.transform.get_transforms(do_flip=False, flip_vert=False, max_rotate=10, max_zoom=1.1, max_lighting=0.2, p_affine=0.85, p_lighting=0.4, max_warp=0.2, xtra_tfms=None) data_test = (fastai.vision.ImageList.from_folder( path).split_none().label_from_folder().transform( tfms, size=size).add_test_folder().databunch(bs=bs).normalize( fastai.vision.imagenet_stats)) with open(save_path + 'test_items.csv', 'w') as myfile: wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) wr.writerow(data_test.test_ds.items) im_res = {} folds = 10 for mod in mods: valid_items = [] for fold in range(folds): train_df = pd.read_csv( 'output/preprocessing/folds/fold_{}.csv'.format(fold), dtype=str) val_df = pd.read_csv( 'output/preprocessing/folds/val_{}.csv'.format(fold), dtype=str) data = (fastai.vision.ImageList.from_df( df=train_df, path=path / 'croped_images', cols='name').split_none().label_from_df( cols='label', classes=classes).transform(tfms, size=size).databunch( bs=bs).normalize(fastai.vision.imagenet_stats)) valid = (fastai.vision.ImageList.from_df( df=val_df, path=path / 'croped_images', cols='name').split_none().label_from_df( cols='label', classes=classes).transform(tfms, size=size).databunch( bs=bs).normalize(fastai.vision.imagenet_stats)) data.valid_dl = valid.train_dl if mod == '50': learn = fastai.vision.learner.cnn_learner( data=data, base_arch=model_50, cut=-2, ps=0.2, metrics=fastai.metrics.accuracy, model_dir='../../training/models') elif mod == '101': learn = fastai.vision.learner.cnn_learner( data=data, base_arch=model_101, cut=-2, ps=0.2, metrics=fastai.metrics.accuracy, model_dir='../../training/models') elif mod == '152': learn = fastai.vision.learner.cnn_learner( data=data, base_arch=model_152, cut=-2, ps=0.2, metrics=fastai.metrics.accuracy, model_dir='../../training/models') elif mod == '161': learn = fastai.vision.learner.cnn_learner( data=data, base_arch=model_161, cut=-1, ps=0.2, metrics=fastai.metrics.accuracy, model_dir='../../training/models') learn.model = nn.DataParallel(learn.model) learn.load('model_' + mod + '_' + version + '_fold_' + str(fold)) learn.data = data_test pr, y = learn.TTA(ds_type=fastai.basic_data.DatasetType.Test) torch.save( pr, save_path + 'test/test_' + mod + '_' + version + '_fold_' + str(fold) + '.pt') learn = None data = None gc.collect() torch.cuda.empty_cache()
def train_50(): mod = str(50) if not os.path.exists(os.path.dirname('output/training/models')): os.makedirs(os.path.dirname('output/training/models')) path = pathlib.PosixPath('output/preprocessing') classes = os.listdir('output/preprocessing/croped_images/') bs = 140 size = 224 version = str(0) tfms = fastai.vision.transform.get_transforms(do_flip=False, flip_vert=False, max_rotate=10, max_zoom=1.1, max_lighting=0.2, p_affine=0.85, p_lighting=0.4, max_warp=0.2, xtra_tfms=None) data_test = (fastai.vision.ImageList.from_folder(path) .split_none() .label_from_folder() .transform(tfms, size=size) .add_test_folder() .databunch(bs=bs) .normalize(fastai.vision.imagenet_stats)) im_res= {} folds = 10 for fold in range(folds): print(fold) train_df = pd.read_csv('output/preprocessing/folds/fold_{}.csv'.format(fold), dtype=str) val_df = pd.read_csv('output/preprocessing/folds/val_{}.csv'.format(fold), dtype=str) data = (fastai.vision.ImageList.from_df(df=train_df, path=path/'croped_images', cols='name') .split_none() .label_from_df(cols='label', classes=classes) .transform(tfms, size=size) .databunch(bs=bs) .normalize(fastai.vision.imagenet_stats)) valid = (fastai.vision.ImageList.from_df(df=val_df, path=path/'croped_images', cols='name') .split_none() .label_from_df(cols='label', classes=classes) .transform(tfms, size=size) .databunch(bs=bs) .normalize(fastai.vision.imagenet_stats)) data.valid_dl = valid.train_dl learn = fastai.vision.learner.cnn_learner( data = data, base_arch=model_50, cut=-2, ps=0.2, metrics=fastai.metrics.accuracy, model_dir='../../training/models') learn.loss_fn = FocalLoss() learn.crit = FocalLoss() learn.model = nn.DataParallel(learn.model) learn.fit_one_cycle(3, max_lr=3e-3) #3 write_log(fold, version, learn, mod) learn.unfreeze() lr = 9e-4 lrs=np.array([lr/10, lr/3]) learn.fit_one_cycle(4, lrs) write_log(fold, version, learn, mod) # lr = 5e-4 # lrs=np.array([lr/10, lr/3]) # learn.fit_one_cycle(4, lrs) # write_log(fold, version, learn, mod) learn.freeze() learn.save('model_'+mod+'_'+version+'_fold_'+str(fold)) learn = None data = None gc.collect() torch.cuda.empty_cache()