def unlabel_prediction(PATH_model, unlabel_name_file):
device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
num_classes = 1
model = UNet11(num_classes=num_classes)
model.to('cuda:1')
model.load_state_dict(torch.load(PATH_model))
model.eval()
######################### setting all data paths#######
outfile_path = 'predictions_VHR/unlabel_test/'
data_path = 'data_VHR'
test_path = "data_VHR/unlabel/" + unlabel_name_file
get_files_path = test_path + "/*.npy"
test_file_names = np.array(sorted(glob.glob(get_files_path)))
###################################
test_transform = DualCompose([CenterCrop(512), ImageOnly(Normalize())])
test_loader = make_loader(test_file_names,
shuffle=False,
transform=test_transform)
metrics = defaultdict(float)
count_img = 0
input_vec = []
pred_vec = []
for inputs, name in test_loader:
inputs = inputs.to(device)
with torch.set_grad_enabled(False):
input_vec.append(inputs.data.cpu().numpy())
pred = model(inputs)
pred = torch.sigmoid(pred)
pred_vec.append(pred.data.cpu().numpy())
count_img += 1
print(count_img)
name_imgs = outfile_path + unlabel_name_file + "_inputs_unlab_" + str(
count_img) + ".npy"
name_preds = outfile_path + unlabel_name_file + "pred_unlab_" + str(
count_img) + ".npy"
np.save(name_imgs, np.array(input_vec))
np.save(name_preds, np.array(pred_vec))
return name_imgs, name_preds
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--device-ids', type=str, default='0', help='For example 0,1 to run on two GPUs')
arg('--fold-out', type=int, help='fold train test', default=0)
arg('--fold-in', type=int, help='fold train val', default=0)
arg('--percent', type=float, help='percent of data', default=1)
arg('--root', default='runs/debug', help='checkpoint root')
arg('--batch-size', type=int, default=4)
arg('--limit', type=int, default=10000, help='number of images in epoch')
arg('--n-epochs', type=int, default=40)
arg('--n-steps', type=int, default=200)
arg('--lr', type=float, default=0.003)
arg('--modelVHR', type=str, default='UNet11', choices=['UNet11','UNet','AlbuNet34','SegNet'])
arg('--dataset-path-HR', type=str, default='data_HR', help='ain path of the HR dataset')
arg('--model-path-HR', type=str, default='logs_HR/mapping/model_40epoch_HR_UNet11.pth', help='path of the model of HR')
arg('--dataset-path-VHR', type=str, default='data_VHR', help='ain path of the VHR dataset')
arg('--name-file-HR', type=str, default='_HR', help='name file of HR dataset')
arg('--dataset-file', type=str, default='VHR', help='main dataset resolution,depend of this correspond a specific crop' )
arg('--out-file', type=str, default='seq', help='the file in which save the outputs')
arg('--train-val-file-HR', type=str, default='train_val_HR', help='name of the train-val file' )
arg('--test-file-HR', type=str, default='test_HR', help='name of the test file' )
arg('--train-val-file-VHR', type=str, default='train_val_850', help='name of the train-val file' )
arg('--test-file-VHR', type=str, default='test_850', help='name of the test file' )
args = parser.parse_args()
root = Path(args.root)
root.mkdir(exist_ok=True, parents=True)
num_classes = 1
input_channels=4
if args.modelVHR == 'UNet11':
model_VHR = UNet11(num_classes=num_classes, input_channels=input_channels)
elif args.modelVHR == 'UNet':
model_VHR = UNet(num_classes=num_classes, input_channels=input_channels)
elif args.modelVHR == 'AlbuNet34':
model_VHR =AlbuNet34(num_classes=num_classes, num_input_channels=input_channels, pretrained=False)
elif args.modelVHR == 'SegNet':
model_VHR = SegNet(num_classes=num_classes, num_input_channels=input_channels, pretrained=False)
else:
model_VHR = UNet11(num_classes=num_classes, input_channels=4)
if torch.cuda.is_available():
if args.device_ids:#
device_ids = list(map(int, args.device_ids.split(',')))
else:
device_ids = None
model_VHR = nn.DataParallel(model_VHR, device_ids=device_ids).cuda()
cudnn.benchmark = True
out_path = Path(('logs_{}/mapping/').format(args.out_file))
#Data-paths:--------------------------VHr-------------------------------------
data_path_VHR = Path(args.dataset_path_VHR)
print("data_path:",data_path_VHR)
name_file_VHR = '_'+ str(int(args.percent*100))+'_percent_'+args.out_file
data_all='data'
##--------------------------------------
############################
# NEstes cross validation K-fold train test
##train_val_file_names, test_file_names_HR = get_split_out(data_path_HR,data_all,args.fold_out)
############################
############################ Cross validation
train_val_file_names=np.array(sorted(glob.glob(str((data_path_VHR/args.train_val_file_VHR/'images'))+ "/*.npy")))
test_file_names_VHR = np.array(sorted(glob.glob(str((data_path_VHR/args.test_file_VHR/'images')) + "/*.npy")))
if args.percent !=1:
extra, train_val_file_names= percent_split(train_val_file_names, args.percent)
train_file_VHR_lab,val_file_VHR_lab = get_split_in(train_val_file_names,args.fold_in)
np.save(str(os.path.join(out_path,"train_files{}_{}_fold{}_{}.npy".format(name_file_VHR, args.modelVHR, args.fold_out, args.fold_in))), train_file_VHR_lab)
np.save(str(os.path.join(out_path,"val_files{}_{}_fold{}_{}.npy". format(name_file_VHR, args.modelVHR, args.fold_out, args.fold_in))), val_file_VHR_lab)
#Data-paths:--------------------------unlabeled VHR-------------------------------------
train_path_VHR_unlab= data_path_VHR/'unlabel'/'train'/'images'
val_path_VHR_unlab = data_path_VHR/'unlabel'/'val'/'images'
train_file_VHR_unlab = np.array(sorted(list(train_path_VHR_unlab.glob('*.npy'))))
val_file_VHR_unlab = np.array(sorted(list(val_path_VHR_unlab.glob('*.npy'))))
print('num train_lab = {}, num_val_lab = {}'.format(len(train_file_VHR_lab), len(val_file_VHR_lab)))
print('num train_unlab = {}, num_val_unlab = {}'.format(len(train_file_VHR_unlab), len(val_file_VHR_unlab)))
max_values_VHR, mean_values_VHR, std_values_VHR=meanstd(train_file_VHR_lab, val_file_VHR_lab,test_file_names_VHR,str(data_path_VHR),input_channels)
def make_loader(file_names, shuffle=False, transform=None,mode='train',batch_size=4, limit=None):
return DataLoader(
dataset=WaterDataset(file_names, transform=transform,mode=mode, limit=limit),
shuffle=shuffle,
batch_size=batch_size,
pin_memory=torch.cuda.is_available()
)
#transformations ---------------------------------------------------------------------------
train_transform_VHR = DualCompose([
CenterCrop(512),
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize(mean=mean_values_VHR,std= std_values_VHR))
])
val_transform_VHR = DualCompose([
CenterCrop(512),
ImageOnly(Normalize(mean=mean_values_VHR, std=std_values_VHR))
])
#-------------------------------------------------------------------
mean_values_HR=(0.11952524, 0.1264638 , 0.13479991, 0.15017026)
std_values_HR=(0.08844988, 0.07304429, 0.06740904, 0.11003125)
train_transform_VHR_unlab = DualCompose([
CenterCrop(512),
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize(mean=mean_values_HR,std= std_values_HR))
])
val_transform_VHR_unlab = DualCompose([
CenterCrop(512),
ImageOnly(Normalize(mean=mean_values_HR, std=std_values_HR))
])
######################## DATA-LOADERS ###########################################################49
train_loader_VHR_lab = make_loader(train_file_VHR_lab, shuffle=True, transform=train_transform_VHR , batch_size = 2, mode = "train")
valid_loader_VHR_lab = make_loader(val_file_VHR_lab, transform=val_transform_VHR, batch_size = 4, mode = "train")
train_loader_VHR_unlab = make_loader(train_file_VHR_unlab, shuffle=True, transform=train_transform_VHR, batch_size = 4, mode = "unlb_train")
valid_loader_VHR_unlab = make_loader(val_file_VHR_unlab, transform=val_transform_VHR, batch_size = 2, mode = "unlb_val")
dataloaders_VHR_lab= {
'train': train_loader_VHR_lab, 'val': valid_loader_VHR_lab
}
dataloaders_VHR_unlab= {
'train': train_loader_VHR_unlab, 'val': valid_loader_VHR_unlab
}
#----------------------------------------------
root.joinpath(('params_{}.json').format(args.out_file)).write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
# Observe that all parameters are being optimized
optimizer_ft = optim.Adam(model_VHR.parameters(), lr= args.lr)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=20, gamma=0.1)
#--------------------------model HR-------------------------------------
PATH_HR= args.model_path_HR
#Initialise the model
model_HR = UNet11(num_classes=num_classes)
model_HR.cuda()
model_HR.load_state_dict(torch.load(PATH_HR))
#---------------------------------------------------------------
model_VHR= utilsTrain_seq.train_model(
out_file=args.out_file,
name_file_VHR=name_file_VHR,
model_HR=model_HR,
model_VHR=model_VHR,
optimizer=optimizer_ft,
scheduler=exp_lr_scheduler,
dataloaders_VHR_lab=dataloaders_VHR_lab,
dataloaders_VHR_unlab=dataloaders_VHR_unlab,
fold_out=args.fold_out,
fold_in=args.fold_in,
name_model_VHR=args.modelVHR,
n_steps=args.n_steps,
num_epochs=args.n_epochs
)
torch.save(model_VHR.module.state_dict(), (str(out_path)+'/model{}_{}_foldout{}_foldin{}_{}epochs.pth').format(args.n_epochs,name_file_VHR,args.modelVHR, args.fold_out,args.fold_in,args.n_epochs))
print(args.modelVHR)
max_values_all_VHR=3521
find_metrics(train_file_names=train_file_VHR_lab,
val_file_names=val_file_VHR_lab,
test_file_names=test_file_names_VHR,
max_values=max_values_all_VHR,
mean_values=mean_values_VHR,
std_values=std_values_VHR,
model=model_VHR,
fold_out=args.fold_out,
fold_in=args.fold_in,
name_model=args.modelVHR,
epochs=args.n_epochs,
out_file=args.out_file,
dataset_file=args.dataset_file,
name_file=name_file_VHR)
def find_metrics(train_file_names,
val_file_names,
test_file_names,
max_values,
mean_values,
std_values,
model,
fold_out='0',
fold_in='0',
name_model='UNet11',
epochs='40',
out_file='VHR',
dataset_file='VHR',
name_file='_VHR_60_fake'):
def make_loader(file_names,
shuffle=False,
transform=None,
limit=None,
mode="train",
batch_size=4,
limite=None):
return DataLoader(dataset=WaterDataset(file_names,
transform=transform,
mode=mode,
limit=limit),
shuffle=shuffle,
batch_size=batch_size,
pin_memory=False)
outfile_path = ('predictions_{}').format(out_file)
f = open(
("predictions_{}/metric{}_{}_foldout{}_foldin{}_{}epochs.txt").format(
out_file, name_file, name_model, fold_out, fold_in, epochs), "w+")
f2 = open(
("predictions_{}/pred_loss_test{}_{}_foldout{}_foldin{}_{}epochs.txt"
).format(out_file, name_file, name_model, fold_out, fold_in,
epochs), "w+")
f.write("Training mean_values:[{}], std_values:[{}] \n".format(
mean_values, std_values))
device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
#####Dilenames ###############################################
print(len(test_file_names))
#####Dataloder ###############################################
if (dataset_file == 'VHR'):
all_transform = DualCompose([
CenterCrop(512),
ImageOnly(Normalize(mean=mean_values, std=std_values))
])
if (dataset_file == 'HR'):
all_transform = DualCompose([
CenterCrop(64),
ImageOnly(Normalize2(mean=mean_values, std=std_values))
])
train_loader = make_loader(train_file_names,
shuffle=True,
transform=all_transform)
val_loader = make_loader(val_file_names, transform=all_transform)
test_loader = make_loader(test_file_names, transform=all_transform)
dataloaders = {
'train': train_loader,
'val': val_loader,
'test': test_loader
}
for phase in ['train', 'val', 'test']:
model.eval()
metrics = defaultdict(float)
############################### train images ###############################
count_img = 0
input_vec = []
labels_vec = []
pred_vec = []
result_dice = []
result_jaccard = []
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
with torch.set_grad_enabled(False):
input_vec.append(inputs.data.cpu().numpy())
labels_vec.append(labels.data.cpu().numpy())
pred = model(inputs)
loss = calc_loss(pred, labels, metrics, 'test')
if phase == 'test':
print_metrics(metrics, f2, 'test')
pred = torch.sigmoid(pred)
pred_vec.append(pred.data.cpu().numpy())
result_dice += [metrics['dice']]
result_jaccard += [metrics['jaccard']]
count_img += 1
print(("{}_{}").format(phase, out_file))
print('Dice = ', np.mean(result_dice), np.std(result_dice))
print('Jaccard = ', np.mean(result_jaccard), np.std(result_jaccard),
'\n')
f.write(("{}_{}\n").format(phase, out_file))
f.write("dice_metric: {:4f}, std: {:4f} \n".format(
np.mean(result_dice), np.std(result_dice)))
f.write("jaccard_metric: {:4f}, std: {:4f} \n".format(
np.mean(result_jaccard), np.std(result_jaccard)))
if phase == 'test':
np.save(
str(
os.path.join(
outfile_path,
"inputs_test{}_{}_foldout{}_foldin{}_{}epochs_{}.npy".
format(name_file, name_model, fold_out, fold_in,
epochs, int(count_img)))), np.array(input_vec))
np.save(
str(
os.path.join(
outfile_path,
"labels_test{}_{}_foldout{}_foldin{}_{}epochs_{}.npy".
format(name_file, name_model, fold_out, fold_in,
epochs, int(count_img)))), np.array(labels_vec))
np.save(
str(
os.path.join(
outfile_path,
"pred_test{}_{}_foldout{}_foldin{}_{}epochs_{}.npy".
format(name_file, name_model, fold_out, fold_in,
epochs, int(count_img)))), np.array(pred_vec))
dataset=ImagesDataset(file_names,channels, transform=transform,mode=mode, limit=limit),
shuffle=shuffle,
batch_size=batch_size,
pin_memory=torch.cuda.is_available()
)
#max_values, mean_values, std_values=meanstd(train_file_names, val_file_names,test_file_names,str(data_path/data_all),input_channels) #_60
print(max_values,mean_values, std_values)
mean_values = [0.485, 0.456, 0.406]
std_values = [0.229, 0.224, 0.225]
train_transform = DualCompose([
CenterCrop(int(args.dataset_file)),
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize(mean=mean_values,std= std_values))
])
val_transform = DualCompose([
CenterCrop(int(args.dataset_file)),
ImageOnly(Normalize(mean=mean_values, std=std_values))
])
train_loader = make_loader(train_file_names,channels, shuffle=True, transform=train_transform, mode='train', batch_size = args.batch_size)
valid_loader = make_loader(val_file_names,channels, transform=val_transform, batch_size = args.batch_size, mode = "train")
dataloaders = {
'train': train_loader, 'val': valid_loader
}
dataloaders_sizes = {
def main():
parser = argparse.ArgumentParser()
arg = parser.add_argument
arg('--device-ids',
type=str,
default='0',
help='For example 0,1 to run on two GPUs')
arg('--fold-out', type=int, default='0', help='fold train-val test')
arg('--fold-in', type=int, default='0', help='fold train val')
arg('--percent', type=float, default=1, help='percent of data')
arg('--root', default='runs/debug', help='checkpoint root')
arg('--batch-size', type=int, default=4, help='HR:4,VHR:8')
arg('--limit', type=int, default=10000, help='number of images in epoch')
arg('--n-epochs', type=int, default=40)
arg('--lr', type=float, default=1e-3)
arg('--model',
type=str,
default='UNet11',
choices=['UNet11', 'UNet', 'AlbuNet34', 'SegNet'])
arg('--dataset-path',
type=str,
default='data_VHR',
help='main file,in which the dataset is: data_VHR or data_HR')
arg('--dataset-file',
type=str,
default='VHR',
help='resolution of the dataset VHR,HR')
#arg('--out-file', type=str, default='VHR', help='the file in which save the outputs')
arg('--train-val-file',
type=str,
default='train_val_850',
help='name of the train-val file VHR:train_val_850 or train_val_HR')
arg('--test-file',
type=str,
default='test_850',
help='name of the test file VHR:test_850 or HR:test_HR')
args = parser.parse_args()
root = Path(args.root)
root.mkdir(exist_ok=True, parents=True)
num_classes = 1
input_channels = 4
if args.model == 'UNet11':
model = UNet11(num_classes=num_classes, input_channels=input_channels)
elif args.model == 'UNet':
model = UNet(num_classes=num_classes, input_channels=input_channels)
elif args.model == 'AlbuNet34':
model = AlbuNet34(num_classes=num_classes,
num_input_channels=input_channels,
pretrained=False)
elif args.model == 'SegNet':
model = SegNet(num_classes=num_classes,
num_input_channels=input_channels,
pretrained=False)
else:
model = UNet11(num_classes=num_classes, input_channels=input_channels)
if torch.cuda.is_available():
if args.device_ids: #
device_ids = list(map(int, args.device_ids.split(',')))
else:
device_ids = None
model = nn.DataParallel(model, device_ids=device_ids).cuda()
cudnn.benchmark = True
####################Change the files_names ######################################
out_path = Path(('logs_{}/mapping/').format(args.dataset_file))
name_file = '_' + str(int(
args.percent * 100)) + '_percent_' + args.dataset_file
data_all = 'data' ##file with all the data
data_path = Path(args.dataset_path)
print("data_path:", data_path)
#################################################################################
# Nested cross validation K-fold train test
#train_val_file_names, test_file_names = get_split_out(data_path,data_all,args.fold_out)
#################################################################################
#eWe are consider the same test in all the cases
train_val_file_names = np.array(
sorted(
glob.glob(
str(data_path / args.train_val_file / 'images') + "/*.npy")))
test_file_names = np.array(
sorted(
glob.glob(str(data_path / args.test_file / 'images') + "/*.npy")))
if args.percent != 1:
extra, train_val_file_names = percent_split(train_val_file_names,
args.percent)
#################################################################################
train_file_names, val_file_names = get_split_in(train_val_file_names,
args.fold_in)
np.save(
str(
os.path.join(
out_path, "train_files{}_{}_fold{}_{}.npy".format(
name_file, args.model, args.fold_out, args.fold_in))),
train_file_names)
np.save(
str(
os.path.join(
out_path,
"val_files{}_{}_fold{}_{}.npy".format(name_file, args.model,
args.fold_out,
args.fold_in))),
val_file_names)
print('num train = {}, num_val = {}'.format(len(train_file_names),
len(val_file_names)))
def make_loader(file_names,
shuffle=False,
transform=None,
mode='train',
batch_size=4,
limit=None):
return DataLoader(dataset=WaterDataset(file_names,
transform=transform,
mode=mode,
limit=limit),
shuffle=shuffle,
batch_size=batch_size,
pin_memory=torch.cuda.is_available())
max_values, mean_values, std_values = meanstd(train_file_names,
val_file_names,
test_file_names,
str(data_path),
input_channels) #_60
print(max_values, mean_values, std_values)
if (args.dataset_file == 'VHR'):
train_transform = DualCompose([
CenterCrop(512),
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize(mean=mean_values, std=std_values))
])
val_transform = DualCompose([
CenterCrop(512),
ImageOnly(Normalize(mean=mean_values, std=std_values))
])
max_values = 3521
train_loader = make_loader(train_file_names,
shuffle=True,
transform=train_transform,
mode='train',
batch_size=args.batch_size) #4 batch_size
valid_loader = make_loader(val_file_names,
transform=val_transform,
batch_size=args.batch_size,
mode="train")
if (args.dataset_file == 'HR'):
train_transform = DualCompose([
CenterCrop(64),
HorizontalFlip(),
VerticalFlip(),
Rotate(),
ImageOnly(Normalize2(mean=mean_values, std=std_values))
])
val_transform = DualCompose([
CenterCrop(64),
ImageOnly(Normalize2(mean=mean_values, std=std_values))
])
train_loader = make_loader(train_file_names,
shuffle=True,
transform=train_transform,
mode='train',
batch_size=args.batch_size) #8 batch_size
valid_loader = make_loader(val_file_names,
transform=val_transform,
mode="train",
batch_size=args.batch_size // 2)
#albunet 34 with only 3 batch_size
dataloaders = {'train': train_loader, 'val': valid_loader}
dataloaders_sizes = {x: len(dataloaders[x]) for x in dataloaders.keys()}
root.joinpath(('params_{}.json').format(args.dataset_file)).write_text(
json.dumps(vars(args), indent=True, sort_keys=True))
optimizer_ft = optim.Adam(model.parameters(), lr=args.lr) #
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft,
step_size=20,
gamma=0.1)
utilsTrain.train_model(dataset_file=args.dataset_file,
name_file=name_file,
model=model,
optimizer=optimizer_ft,
scheduler=exp_lr_scheduler,
dataloaders=dataloaders,
fold_out=args.fold_out,
fold_in=args.fold_in,
name_model=args.model,
num_epochs=args.n_epochs)
torch.save(
model.module.state_dict(),
(str(out_path) + '/model{}_{}_foldout{}_foldin{}_{}epochs').format(
name_file, args.model, args.fold_out, args.fold_in, args.n_epochs))
print(args.model)
find_metrics(train_file_names=train_file_names,
val_file_names=val_file_names,
test_file_names=test_file_names,
max_values=max_values,
mean_values=mean_values,
std_values=std_values,
model=model,
fold_out=args.fold_out,
fold_in=args.fold_in,
name_model=args.model,
epochs=args.n_epochs,
out_file=args.dataset_file,
dataset_file=args.dataset_file,
name_file=name_file)