def __getitem__(self, index):
path = os.path.join(self.data_path, self.samples[index])
#img = default_loader(path)
img = tif_loader(path)
#print('Test ', np.array(img).shape)
if self.random_crop:
# imgw, imgh = img.size
# if imgh < self.image_shape[0] or imgw < self.image_shape[1]:
# img = transforms.Resize(min(self.image_shape))(img)
# img = transforms.RandomCrop(self.image_shape)(img)
h, w, _ = img.shape
targetH, targetW = self.image_shape[0], self.image_shape[1]
randomH = np.random.randint(0, h - targetH)
randomW = np.random.randint(0, w - targetW)
imgCrop = img[randomH:(randomH + targetH),
randomW:(randomW + targetW), :]
imgCrop = np.transpose(imgCrop, [2, 0, 1])
else:
img = transforms.Resize(self.image_shape)(img)
img = transforms.RandomCrop(self.image_shape)(img)
imgTensor = torch.from_numpy(imgCrop)
#img = transforms.ToTensor()(img) # turn the image to a tensor
img = normalize(imgTensor)
#img = transfer2tensor(img)
if self.return_name:
return self.samples[index], img
else:
return img
def __getitem__(self, index):
file_idx = np.where(self.samples > index)[0][0]
if file_idx > 0:
index -= self.samples[file_idx - 1]
with h5py.File(self.mouse_files[file_idx], 'r') as f:
img = np.uint8(f['frames'][index] / 100 * 255)
img = img[:, :, None]
img = Image.fromarray(np.tile(img, (1, 1, 3)))
if self.random_crop:
imgw, imgh = img.size
if imgh < self.image_shape[0] or imgw < self.image_shape[1]:
img = transforms.Resize(min(self.image_shape))(img)
img = transforms.RandomCrop(self.image_shape)(img)
else:
img = transforms.Resize(self.image_shape)(img)
img = transforms.RandomCrop(self.image_shape)(img)
img = transforms.ToTensor()(img) # turn the image to a tensor
img = normalize(img)
if self.return_name:
return self.mouse_files[file_idx], img
else:
return img
def _get_generated_image(
x,
mask=None,
):
# global generated_image_idx
global netG
if mask is None:
mask = np.zeros(x.shape, dtype=np.uint8)
if len(x.shape) == 2:
x = PIL.Image.fromarray(np.stack((x, ) * 3, axis=-1))
elif len(x.shape) == 3 and x.shape[-1] == 3:
x = PIL.Image.fromarray(x)
else:
print(x.shape)
print("dim error")
import sys
sys.exit(0)
mask = PIL.Image.fromarray(mask)
x = transforms.Resize(config['image_shape'][:-1])(x)
x = transforms.CenterCrop(config['image_shape'][:-1])(x)
mask = transforms.Resize(config['image_shape'][:-1])(mask)
mask = transforms.CenterCrop(config['image_shape'][:-1])(mask)
x = transforms.ToTensor()(x)
mask = transforms.ToTensor()(mask)[0].unsqueeze(dim=0)
x = normalize(x)
x = x * (1. - mask)
x = x.unsqueeze(dim=0)
mask = mask.unsqueeze(dim=0)
if cuda:
# device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
x = x.cuda()
mask = mask.cuda()
# Inference
x1, x2, offset_flow, feature = netG(x, mask)
inpainted_result = x2 * mask + x * (1. - mask)
if cuda:
inpainted_result = inpainted_result.cpu()
np_inpainted_result = np.rollaxis(
np.uint8(np.squeeze(inpainted_result.detach().numpy(), axis=0)), 0, 3)
assert np_inpainted_result.shape == (256, 256, 3)
# return 255-np_inpainted_result
# vutils.save_image(inpainted_result, args.output, padding=0, normalize=True)
# from PIL import Image
grid = vutils.make_grid(inpainted_result)
# Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
ndarr = grid.mul_(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to(
'cpu', torch.uint8).numpy()
return ndarr, feature
def train(self, X, Y, G_X, G_Y):
'''
:param X: placeholder for X
:param Y: placeholder for Y
:param G_X: tfrecords generator X
:param G_Y: tfrecords generator Y
:param G_name: tfrecords generator name
:return: null
'''
self.model(X).optimizer(Y)
with tf.Session() as session:
self.writer = tf.summary.FileWriter(logdir=self.writer_path,
graph=session.graph)
if not self.has_model():
session.run(tf.global_variables_initializer())
else:
print("=> loading session from: {:s}".format(self.save_path))
self.saver.restore(session, self.save_path)
tf.train.start_queue_runners(sess=session,
coord=tf.train.Coordinator())
global_step = session.graph.get_tensor_by_name(
"leaning_rate/global_step:0")
for index in range(global_step.eval(), self._conf.total_batch):
(samples, labels) = session.run([G_X, G_Y])
# _X = np.reshape(G.normalize(samples), [self.batch_size, self.image_size * self.image_size * 3])
(_X, _Y) = (utils.normalize(samples),
utils.reformat(labels, self._conf.label_size))
(_, params, predict, loss, accuracy, output,
summary) = session.run([
self.optimizer, self.params_value, self.predict,
self.loss, self.accuracy, self.output, self.merged_summary
],
feed_dict={
self.X: _X,
self.Y: _Y
})
if index % self._conf.train["summary_steps"] == 0:
self.writer.add_summary(summary, index)
self.writer.flush()
if index % self._conf.train["saving_steps"] == 0:
for i in range(len(self.params_key)):
print("=> {:s} -> {:s}".format(self.params_key[i],
str(params[i][0])))
print("=> step={:d}, loss={:.5f}, accuracy:{:.3f}%".format(
index, loss, accuracy))
if len(params) > 0:
print("\n\n\n")
self.saver.save(session, self.save_path)
def __getitem__(self, index):
path = os.path.join(self.data_path, self.samples[index])
img = default_loader(path)
if self.random_crop:
imgw, imgh = img.size
if imgh < self.image_shape[0] or imgw < self.image_shape[1]:
img = transforms.Resize(min(self.image_shape))(img)
img = transforms.RandomCrop(self.image_shape)(img)
else:
img = transforms.Resize(self.image_shape)(img)
img = transforms.RandomCrop(self.image_shape)(img)
img = transforms.ToTensor()(img) # turn the image to a tensor
img = normalize(img)
if self.return_name:
return self.samples[index], img
else:
return img
def iterator_train(self, X, Y, iterator):
self.model(X).optimizer(Y)
with tf.Session() as session:
self.writer = tf.summary.FileWriter(logdir=self.writer_path,
graph=session.graph)
if not self.has_model():
session.run(tf.global_variables_initializer())
else:
print("=> loading session from: {:s}".format(self.save_path))
self.saver.restore(session, self.save_path)
global_step = session.graph.get_tensor_by_name(
"leaning_rate/global_step:0")
for index in range(global_step.eval(), self._conf.total_batch):
for (count, step_count, samples, labels) in iterator():
(_X, _Y) = (utils.normalize(samples),
utils.reformat(labels, self._conf.label_size))
(_, params, predict, loss, accuracy, output,
summary) = session.run([
self.optimizer, self.params_value, self.predict,
self.loss, self.accuracy, self.output,
self.merged_summary
],
feed_dict={
self.X: _X,
self.Y: _Y
})
if step_count % self._conf.train["summary_steps"] == 0:
self.writer.add_summary(summary, index)
self.writer.flush()
if step_count % self._conf.train["saving_steps"] == 0:
for i in range(len(self.params_key)):
print("=> {:s} -> {:s}".format(
self.params_key[i], str(params[i][0])))
print("=> step={:d}, loss={:.5f}, accuracy:{:.3f}%".
format(index, loss, accuracy))
if len(params) > 0:
print("\n\n\n")
self.saver.save(session, self.save_path)
def main():
args = parser.parse_args()
config = get_config(args.config)
# CUDA configuration
cuda = config['cuda']
device_ids = config['gpu_ids']
if cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(
str(i) for i in device_ids)
device_ids = list(range(len(device_ids)))
config['gpu_ids'] = device_ids
cudnn.benchmark = True
print("Arguments: {}".format(args))
# Set random seed
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random seed: {}".format(args.seed))
random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed_all(args.seed)
print("Configuration: {}".format(config))
try: # for unexpected error logging
with torch.no_grad(): # enter no grad context
if is_image_file(args.image):
if args.mask and is_image_file(args.mask):
# Test a single masked image with a given mask
x = default_loader(args.image)
mask = default_loader(args.mask)
x = transforms.Resize(config['image_shape'][:-1])(x)
x = transforms.CenterCrop(config['image_shape'][:-1])(x)
mask = transforms.Resize(config['image_shape'][:-1])(mask)
mask = transforms.CenterCrop(
config['image_shape'][:-1])(mask)
x = transforms.ToTensor()(x)
mask = transforms.ToTensor()(mask)[0].unsqueeze(dim=0)
x = normalize(x)
x = x * (1. - mask)
x = x.unsqueeze(dim=0)
mask = mask.unsqueeze(dim=0)
elif args.mask:
raise TypeError(
"{} is not an image file.".format(args.mask))
else:
# Test a single ground-truth image with a random mask
ground_truth = default_loader(args.image)
ground_truth = transforms.Resize(
config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.CenterCrop(
config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.ToTensor()(ground_truth)
ground_truth = normalize(ground_truth)
ground_truth = ground_truth.unsqueeze(dim=0)
bboxes = random_bbox(
config, batch_size=ground_truth.size(0))
x, mask = mask_image(ground_truth, bboxes, config)
# Set checkpoint path
if not args.checkpoint_path:
checkpoint_path = os.path.join('checkpoints',
config['dataset_name'],
config['mask_type'] + '_' + config['expname'])
else:
checkpoint_path = args.checkpoint_path
# Define the trainer
netG = Generator(config['netG'], cuda, device_ids)
# Resume weight
last_model_name = get_model_list(
checkpoint_path, "gen", iteration=args.iter)
netG.load_state_dict(torch.load(last_model_name))
model_iteration = int(last_model_name[-11:-3])
print("Resume from {} at iteration {}".format(
checkpoint_path, model_iteration))
if cuda:
netG = nn.parallel.DataParallel(
netG, device_ids=device_ids)
x = x.cuda()
mask = mask.cuda()
# Inference
x1, x2, offset_flow = netG(x, mask)
inpainted_result = x2 * mask + x * (1. - mask)
vutils.save_image(inpainted_result, args.output,
padding=0, normalize=True)
print("Saved the inpainted result to {}".format(args.output))
if args.flow:
vutils.save_image(offset_flow, args.flow,
padding=0, normalize=True)
print("Saved offset flow to {}".format(args.flow))
else:
raise TypeError("{} is not an image file.".format)
# exit no grad context
except Exception as e: # for unexpected error logging
print("Error: {}".format(e))
raise e
from utils.tools import get_word_to_index, tokenize, normalize
from utils.resourceManager import get_data
from utils.statsmanager import StatsManager
print("Getting data...")
data = get_data("train", "zh")
a_to_index = get_word_to_index([d[0] for d in data])
b_to_index = get_word_to_index([d[1] for d in data])
data = [(tokenize(a, a_to_index), tokenize(b, b_to_index), score)
for a, b, score in data]
val_data = get_data("dev", "zh")
val_data = [(tokenize(a, a_to_index), tokenize(b, b_to_index), score)
for a, b, score in val_data]
val_a_normalized, val_a_len = normalize([row[0] for row in val_data])
val_b_normalized, val_b_len = normalize([row[1] for row in val_data])
val_a = torch.tensor(val_a_normalized, dtype=int)
val_b = torch.tensor(val_b_normalized, dtype=int)
val_labels = torch.tensor([row[2] for row in val_data]).view(
(len(val_data), 1))
print("Tokenized data")
model = LSTM(a_vocab_size=len(a_to_index),
b_vocab_size=len(b_to_index),
padding_index=0,
lstms_in_out=((5, 5), (5, 5)),
linear_layers=(10, 5),
out_size=1,
hidden_activation=nn.ReLU,
def test(self):
netG=Generator().to(device)
netD=Discriminator().to(device)
path = os.path.join(self.dataset_path, 'test')
print(path, 'PATH')
os.getcwd()
path2 = self.choose_net
beta1=0.5
beta2=0.999
Testset = torchvision.datasets.DatasetFolder(
root=path,
loader=npy_loader,
extensions=('.npy',)
)
test_loader = torch.utils.data.DataLoader(dataset=Testset,
batch_size=1,
shuffle=False)
g=0
g_optimizer = torch.optim.Adam(netG.parameters(), 0.0001, [beta1, beta2])
d_optimizer = torch.optim.Adam(netD.parameters(), 0.0001, [beta1, beta2])
netG, g_optimizer = amp.initialize(netG, g_optimizer, opt_level='O1')
netD, d_optimizer = amp.initialize(netD, d_optimizer, opt_level='O1')
g_optimizer.zero_grad()
d_optimizer.zero_grad()
netG = torch.nn.DataParallel(netG, device_ids=[id], output_device=id)
netD = torch.nn.DataParallel(netD, device_ids=[id], output_device=id)
try_loading_file=True
if try_loading_file:
try:
netG.load_state_dict(torch.load(os.path.join(path2, 'netG_synthetic.pt'), map_location={'cuda:0': 'cpu'})) # does NOT load optimizer state etc.
netD.load_state_dict(torch.load(os.path.join(path2, 'netD_synthetic.pt'), map_location={'cuda:0': 'cpu'}))
print("loaded model from file")
except:
print("loading model from file failed; created new model")
c_dim=2
#
# =================================================================================== #
# 5. Testing #
# =================================================================================== #
"""Translate images using StarGAN trained on a single dataset."""
loss_metric = nn.MSELoss()
with torch.no_grad():
sum_dice = 0;
count = 0
correct = 0
total = 0; total_auc=0
netD = netD.eval(); netG = netG.train()
total_rec = 0;total_diff = 0; total_var=0; total_var2=0;sum_ssim=0;threshtot=0
count_diseased = 0;count_healthy = 0
for i, (X3, c_org) in enumerate(test_loader):
GT = -X3[:, 1, :, :]
thresh = threshold_otsu(np.array(abs(GT[0,:,:])))
threshtot+=thresh
x_real = torch.tensor(X3[:, :1, :, :]).to(device)
noise = torch.rand(x_real.shape).to(device)*0
x_real = (x_real + 0.05 * noise).half()
(_, out_cls) = netD(x_real,g)
_, predicted = torch.max(out_cls.data, 1)
correct += (predicted.cpu() == c_org).sum().item()
total+=1
avg_thresh=0.44195
c_trg_list = create_labels(c_org, c_dim)
# # Translate images.
x_fake_list = [x_real]
for c_trg in c_trg_list:
x_fake_list.append((netG(x_real, c_trg)))
#
if c_org == 0:
diff = normalize(x_fake_list[0][0, 0, :, :]).cpu() - normalize (x_fake_list[1][0, 0, :,:]).cpu() # check whether 2 ist diseased (ich glaub schon). Dann nimm immer die Differenz zwischen diseased und Original
thresholded_images = np.double(abs(diff) > avg_thresh)*1
GTthresh = np.double(abs(GT )> avg_thresh) * 1
reconstruction = loss_metric(normalize(x_fake_list[1]), normalize(x_fake_list[0]))
varianz=diff.var()
total_rec += reconstruction
total_var+=varianz
count_healthy += 1
else:
diff = normalize(x_fake_list[0][0, 0, :, :]).cpu() - normalize(x_fake_list[1][0, 0, :, :]).cpu() # check whether 2 ist diseased (ich glaub schon). Dann nimm immer die Differenz zwischen diseased und Original
thresh = threshold_otsu(np.array(abs(diff)))
print(i, thresh, 2 * abs(thresh))
thresholded_images = np.double(abs(diff) > avg_thresh)*1#1 * abs(thresh)) * 1
GTthresh = np.double(abs(GT) > avg_thresh) * 1
region = loss_metric(diff, GT[0, :, :])
total_diff += region
varianz2 = (normalize(np.array(GT[0, :, :])) - normalize(np.array(diff))).var()
total_var2 += varianz2
count_diseased += 1
(output_DSC, avg) = eval_binary_classifier(np.array(GTthresh[0, :, :]), thresholded_images)
sum_dice += output_DSC['DSC'];
count += 1
ssim_val = ssim(visualize(diff[None,None,...]), visualize(GT[None,...]), data_range=1, size_average=False)
sum_ssim+=ssim_val
pixel_wise_cls = np.array(torch.tensor(visualize(abs(diff))).view(1, -1))[0, :]
pixel_wise_gt = np.array(torch.tensor(GTthresh).view(1, -1))[0, :]
auc = roc_auc_score(pixel_wise_gt, pixel_wise_cls)
print('auc', i, auc)
total_auc += auc
#
if i%10==0:
plt.figure(i) #plot results
ax = plt.subplot(2, 4, 1)
plt.imshow((normalize((x_fake_list[0][0, 0, :, :]))).cpu())
ax.title.set_text('original')
ax = plt.subplot(2, 4, 2)
plt.imshow(normalize(x_fake_list[1][0, 0, :, :]).cpu())
ax.title.set_text('label 0')
ax.axis('off')
ax = plt.subplot(2, 4, 3)
plt.imshow(normalize(x_fake_list[2][0, 0, :, :]).cpu())
ax.axis('off')
ax.title.set_text('label 1')
ax = plt.subplot(2, 4, 5)
plt.imshow(thresholded_images)
ax.title.set_text('differenz thresholded')
ax = plt.subplot(2, 4, 6)
plt.imshow(GT[0, :, :])
#
ax.title.set_text('Ground Truth')
ax = plt.subplot(2, 4, 4)
plt.imshow(diff)
ax.title.set_text('difference')
if c_org != 0:
plt.suptitle(auc)
ax = plt.subplot(2, 4, 8)
plt.imshow(GTthresh[0, :, :])
ax.title.set_text('GTthresh')
accuracy= 100 * correct / total
avg_diff=total_diff/count_diseased
avg_auc=total_auc/count_diseased
avg_rec=total_rec/count_healthy
avg_var=total_var/count_healthy
avg_var2=total_var2/count_diseased
avg_ssim=sum_ssim/count_diseased
avg_dice=sum_dice/count
print('average mse reconstruction error', avg_rec,'average mse in segmentation', avg_diff, 'average Dice' ,avg_dice, 'classification accuracy', accuracy, 'AUROC', avg_auc, 'SSIM', avg_ssim,'varianz healthy', avg_var,'varianz krak', avg_var2 )
f = open('./descargan.txt', 'w')
f.write('auroc '+str(auc)+'\n')
f.write('MSE(a_h, r_h) ' + str(avg_rec) + '\n')
f.write('varianz reconstruction ' + str(avg_var) + '\n')
f.write('varianz difference ' + str(avg_var2) + '\n')
f.write('Dice ' + str(avg_dice) + '\n')
f.write('classification accuracy ' + str(accuracy) + '\n')
f.write('MSE(gt, d) ' + str(avg_diff) + '\n')
f.write('SSIM ' + str(avg_ssim) + '\n')
def generateInpaintedImage(args, netG, imagePath):
config = get_config(args.g_config)
occlusions = []
# CUDA configuration
cuda = config['cuda']
device_ids = config['gpu_ids']
if cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(
str(i) for i in device_ids)
device_ids = list(range(len(device_ids)))
config['gpu_ids'] = device_ids
cudnn.benchmark = True
print("Arguments: {}".format(args))
# Set random seed
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random seed: {}".format(args.seed))
random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed_all(args.seed)
try: # for unexpected error logging
with torch.no_grad(): # enter no grad context
if is_image_file(imagePath):
if args.mask and is_image_file(args.mask):
# Test a multiple masked image with a given mask
x = default_loader(imagePath)
x = transforms.Resize([512, 1024])(x)
mask = default_loader(args.mask)
mask = transforms.Resize(config['image_shape'][:-1])(mask)
mask = transforms.CenterCrop(
config['image_shape'][:-1])(mask)
mask = transforms.ToTensor()(mask)[0].unsqueeze(dim=0)
mask = mask.unsqueeze(dim=0)
w, h = x.size
first = x.crop((0, 0, w // 3, h))
second = x.crop((w // 3, 0, ((w // 3) * 2) + 2, h))
third = x.crop(((w // 3) * 2, 0, w, h))
for y in [first, second, third]:
y = transforms.CenterCrop(
config['image_shape'][:-1])(y)
y = transforms.ToTensor()(y)
y = normalize(y)
y = y * (1. - mask)
occlusions.append(y)
elif args.mask:
raise TypeError("{} is not an image file.".format(
args.mask))
default_image = default_loader(imagePath)
di_w, di_h = default_image.size
for idx, occlusion in enumerate(occlusions):
if cuda:
occlusion = occlusion.cuda()
mask = mask.cuda()
# Inference
x1, x2, offset_flow = netG(occlusion, mask)
inpainted_result = x2 * mask + occlusion * (1. - mask)
inp_hw = config['image_shape'][1]
if idx == 0:
offset = ((di_w // 3 - inp_hw) // 2,
(di_h - inp_hw) // 2)
elif idx == 1:
offset = ((di_w - inp_hw) // 2, (di_h - inp_hw) // 2)
elif idx == 2:
offset = ((((di_w - inp_hw) // 2) + (di_w // 3)),
(di_h - inp_hw) // 2)
grid = vutils.make_grid(inpainted_result, normalize=True)
# Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
ndarr = grid.mul_(255).add_(0.5).clamp_(0, 255).permute(
1, 2, 0).to('cpu', torch.uint8).numpy()
im = Image.fromarray(ndarr)
im = transforms.CenterCrop(config['mask_shape'])(im)
im = transforms.Resize(config['image_shape'][:-1])(im)
default_image.paste(im, offset)
return default_image
else:
raise TypeError("{} is not an image file.".format)
# exit no grad context
except Exception as e: # for unexpected error logging
print("Error: {}".format(e))
raise e
def main():
args = parser.parse_args()
config = get_config(args.config)
# CUDA configuration
cuda = config['cuda']
device_ids = config['gpu_ids']
if cuda:
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(
str(i) for i in device_ids)
device_ids = list(range(len(device_ids)))
config['gpu_ids'] = device_ids
cudnn.benchmark = True
print("Arguments: {}".format(args))
# Set random seed
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random seed: {}".format(args.seed))
random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed_all(args.seed)
print("Configuration: {}".format(config))
try: # for unexpected error logging
with torch.no_grad(): # enter no grad context
file = dataset_files(args.test_root, "*.jpg")
mask_file = dataset_files(args.mask_dir, "*.png")
for j in range(len(mask_file)):
for i in range(len(file)):
if is_image_file(file[i]):
if mask_file and is_image_file(mask_file[j]):
# Test a single masked image with a given mask
x = default_loader(file[i])
mask = default_loader(mask_file[j])
# x = cv2.cvtColor(cv2.imread(file[i]), cv2.COLOR_BGR2RGB)
# mask = cv2.cvtColor(cv2.imread(mask_file[j]), cv2.COLOR_BGR2RGB)
# x = cv2.resize(x, (config['image_shape'][0], config['image_shape'][1]))
# mask = cv2.resize(mask, (config['image_shape'][0], config['image_shape'][1]))
x = transforms.Resize(
config['image_shape'][:-1])(x)
x = transforms.CenterCrop(
config['image_shape'][:-1])(x)
# mask = transforms.Resize(config['image_shape'][:-1])(mask)
# mask = transforms.CenterCrop(config['image_shape'][:-1])(mask)
x = transforms.ToTensor()(x)
mask = transforms.ToTensor()(mask)[0].unsqueeze(
dim=0)
x = normalize(x)
x = x * (1. - mask)
x = x.unsqueeze(dim=0)
# x_raw = x
mask = mask.unsqueeze(dim=0)
elif mask_file[j]:
raise TypeError("{} is not an image file.".format(
mask_file[j]))
else:
# Test a single ground-truth image with a random mask
ground_truth = default_loader(file[i])
ground_truth = transforms.Resize(
config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.CenterCrop(
config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.ToTensor()(ground_truth)
ground_truth = normalize(ground_truth)
ground_truth = ground_truth.unsqueeze(dim=0)
bboxes = test_bbox(config,
batch_size=ground_truth.size(0),
t=50,
l=50)
x, mask = mask_image(ground_truth, bboxes, config)
# Set checkpoint path
if not args.checkpoint_path:
checkpoint_path = os.path.join(
'checkpoints', config['dataset_name'],
config['mask_type'] + '_' + config['expname'])
else:
checkpoint_path = args.checkpoint_path
# Define the trainer
netG = Generator(config['netG'], cuda, device_ids)
# Resume weight
g_checkpoint = torch.load(f'{checkpoint_path}/gen.pt')
netG.load_state_dict(g_checkpoint)
# model_iteration = int(last_model_name[-11:-3])
print("Model Resumed".format(checkpoint_path))
if cuda:
netG = nn.parallel.DataParallel(
netG, device_ids=device_ids)
x = x.cuda()
mask = mask.cuda()
# Inference
x1, x2 = netG(x, mask)
inpainted_result = x2 * mask + x * (1. - mask)
inpainted_result_cpu = torch.Tensor.cpu(
inpainted_result).detach().permute(0, 2, 3, 1)
inpainted_result_cpu = np.asarray(
inpainted_result_cpu[0])
inpainted_result_cpu = cv2.normalize(
inpainted_result_cpu, inpainted_result_cpu, 0, 255,
cv2.NORM_MINMAX)
# cat_result = torch.cat([x, inpainted_result, ground_truth], dim=3).cuda()
vutils.save_image(inpainted_result,
args.output_dir +
'output_{}/'.format(j + 1) +
'output_{}.png'.format(i),
padding=0,
normalize=True)
# cv2.imwrite(args.output_dir+ 'output_{}/'.format(j+1) + 'output_{}.png'.format(i), inpainted_result_cpu)
# cv2.cvtColor(inpainted_result_cpu, cv2.COLOR_BGR2RGB))
print("{}th image saved".format(i))
else:
raise TypeError("{} is not an image file.".format)
# exit no grad context
except Exception as e: # for unexpected error logging
print("Error: {}".format(e))
raise e
def test(self):
path = os.path.join(self.dataset_path, 'test')
print(path, 'PATH')
path2 = self.choose_net
netG = Generator().to(device)
netD = Discriminator().to(device)
p1 = np.array([np.array(p.shape).prod()
for p in netG.parameters()]).sum()
p2 = np.array([np.array(p.shape).prod()
for p in netD.parameters()]).sum()
print(p1, p2, p1 / (256 * 256))
beta1 = 0.5
beta2 = 0.999
Testset = torchvision.datasets.DatasetFolder(root=path,
loader=npy_loader,
extensions=('.npy', ))
test_loader = torch.utils.data.DataLoader(dataset=Testset,
batch_size=1,
shuffle=False)
g_optimizer = torch.optim.Adam(netG.parameters(), 0.0001,
[beta1, beta2])
d_optimizer = torch.optim.Adam(netD.parameters(), 0.0001,
[beta1, beta2])
netG, g_optimizer = amp.initialize(netG, g_optimizer, opt_level='O1')
netD, d_optimizer = amp.initialize(netD, d_optimizer, opt_level='O1')
netG = torch.nn.DataParallel(netG, device_ids=[id], output_device=id)
netD = torch.nn.DataParallel(netD, device_ids=[id], output_device=id)
g = 0
try_loading_file = True
if try_loading_file:
try:
netG.load_state_dict(
torch.load(os.path.join(path2, 'netG_chexpert.pt'),
map_location={
'cuda:0': 'cpu'
})) #does NOT load optimizer state etc.
netD.load_state_dict(
torch.load(os.path.join(path2, 'netD_chexpert.pt'),
map_location={
'cuda:0': 'cpu'
})) #does NOT load optimizer state etc.
print("loaded model from file")
except:
print("loading model from file failed; created new model")
c_dim = 2
# =================================================================================== #
# 5. Testing #
# =================================================================================== #
"""Translate images using StarGAN trained on a single dataset."""
netD = netD.eval()
total = 0
correct = 0
count_gesund = 0
total_rec = 0
total_var = 0
loss_metric = nn.MSELoss()
with torch.no_grad():
long_pred = torch.zeros(0).long()
long_cls = torch.zeros(0).long()
long_score = torch.zeros(0)
for i, (X, label_org) in enumerate(test_loader):
if i < 500:
# Prepare input images and target domain labels.
x_real = np.array(X).astype(np.float32)
x_real = np.transpose(np.array(x_real), (0, 3, 1, 2))
x_real = torch.tensor(x_real).half().to(device)
x_real = normalize(x_real).to(device)
c_trg_list = create_labels(label_org, c_dim)
print('xreal', x_real.shape)
(_, out_cls) = netD(x_real, 0)
print('out_cls', out_cls)
_, predicted = torch.max(out_cls.data, 1)
y_score = out_cls[:, 1].cpu()
total += 1
correct += (predicted.cpu() == label_org).sum().item()
long_pred = torch.cat((long_pred, predicted.cpu()), dim=0)
long_cls = torch.cat((long_cls, label_org), dim=0)
long_score = torch.cat((long_score, y_score), dim=0)
# Translate images.
x_fake_list = [x_real]
for c_trg in c_trg_list:
x_fake_list.append((netG(x_real, c_trg)))
if label_org == 0:
text = 'original healthy'
reconstruction = loss_metric(x_fake_list[1],
x_fake_list[0])
total_rec += reconstruction
count_gesund += 1
diff = normalize(
x_fake_list[1][0, 0, :, :]).cpu() - normalize(
x_fake_list[0][0, 0, :, :]).cpu()
thresh = threshold_otsu(np.array(diff))
print(i, thresh)
varianz = diff.var()
total_var += varianz
else:
text = 'original diseased'
diff = -normalize(
x_fake_list[1][0, 0, :, :]).cpu() + normalize(
x_fake_list[0][0, 0, :, :]).cpu()
thresh = threshold_otsu(np.array(abs(diff)))
thresholded_images = np.double(
abs(diff) > 1 * abs(thresh))
print(i, thresh)
if i % 2 == 0 and label_org == 1: #plot output images
img = torch.zeros(7, 256, 256)
img[0, :, :] = x_fake_list[0][0, 0, :, :].cpu()
img[1, :, :] = x_fake_list[1][0, 0, :, :].cpu()
img[2, :, :] = x_fake_list[2][0, 0, :, :].cpu()
img[3, :, :] = diff
img[4, :, :] = torch.tensor(thresholded_images)
plt.figure(i)
ax = plt.subplot(2, 3, 1)
plt.imshow((normalize(
(x_fake_list[0][0, 0, :, :]))).cpu())
ax.axis('off')
ax.title.set_text(text)
ax = plt.subplot(2, 3, 2)
plt.imshow(normalize(x_fake_list[1][0, 0, :, :]).cpu())
ax.title.set_text('generated healthy')
ax.axis('off')
ax = plt.subplot(2, 3, 3)
plt.imshow(normalize(x_fake_list[2][0, 0, :, :]).cpu())
ax.axis('off')
ax.title.set_text('generated diseased')
ax = plt.subplot(2, 3, 4)
plt.imshow(diff)
ax.title.set_text('difference')
ax.axis('off')
accuracy = 100 * correct / total
auc = roc_auc_score(long_cls, long_score)
(kappa, upper, lower) = kappa_score(long_pred, long_cls)
avg_rec = total_rec / count_gesund
avg_var = total_var / count_gesund
print('AUROC', auc, 'accuracy', accuracy, 'kappa', kappa,
'mse reconstruction error', avg_rec, 'varianz gesund', avg_var)
f = open('.descargan.txt', 'w')
f.write('auroc ' + str(auc) + '\n')
f.write('accuracy ' + str(accuracy) + '\n')
f.write('MSE(a_h, r_h) ' + str(avg_rec) + '\n')
f.write('kappa ' + str(kappa) + '\n')
f.write('varianz reconstruction ' + str(avg_var) + '\n')
elif (label in [
"Silence", "IPU", "SpeechActivity", "Overlap", "joint_laugh"
]):
df[label] = sample_square(time_series[label], physio_index)
elif label in [
"Particles", "Discourses", "FilledBreaks", "Laughters",
"Feedbacks", "UnionSocioItems"
]:
df[label] = sample_cont_ts(time_series[label],
physio_index,
mode="binary")
else:
if label == "Signal":
df[label] = ts.normalize(
sample_cont_ts(time_series[label], physio_index))
else:
df[label] = sample_cont_ts(time_series[label], physio_index)
if args.left:
for i in range(len(labels)):
labels[i] += "_P"
else:
for i in range(len(labels)):
labels[i] += "_I"
df.columns = ["Time (s)"] + labels
# save data
df.to_pickle(output_filename_pkl)
if is_image_file(args.image):
if args.mask and is_image_file(args.mask):
# Test a single masked image with a given mask
x = tif_loader(args.image)
## center crop
x = x[110:366, 110:366, :]
x = torch.from_numpy(x)
mask = default_loader(args.mask) ## 476 --> 256
#x = x[110:366, 110:366,:]
#x = transforms.Resize(config['image_shape'][:-1])(x)
#x = transforms.CenterCrop(config['image_shape'][:-1])(x)
mask = transforms.Resize(config['image_shape'][:-1])(mask)
mask = transforms.CenterCrop(config['image_shape'][:-1])(mask)
#x = transforms.ToTensor()(x)
mask = transforms.ToTensor()(mask)[0].unsqueeze(dim=0)
x = normalize(x)
#x = x * (1. - mask)
x = x.unsqueeze(dim=0)
mask = mask.unsqueeze(dim=0)
elif args.mask:
raise TypeError("{} is not an image file.".format(args.mask))
else:
# Test a single ground-truth image with a random mask
ground_truth = tif_loader(args.image)
#ground_truth = transforms.Resize(config['image_shape'][:-1])(ground_truth)
#ground_truth = transforms.CenterCrop(config['image_shape'][:-1])(ground_truth)
#ground_truth = transfer2tensor(ground_truth)
#ground_truth = transforms.ToTensor()(ground_truth)
x = ground_truth[110:366, 110:366, :]
x = np.transpose(x, [2, 0, 1])
#print('Output min, max',x.min(),x.max())
def __getitem__(self, index):
# first, load gt image
gt_path = os.path.join(self.gt_path, self.samples[index])
img = default_loader(gt_path, chan='L')
# get original image for GT images
# png_to_jpg_ext = self.samples[index]
original_path = os.path.join(self.data_path, self.samples[index])
original_path = original_path.replace('png', 'jpg')
# but, if the original image is not exist, skip it
orig_img = default_loader(original_path)
#img_path = os.path.join(self.data_path, self.gt_samples[index])
#print("====img(labeled)")
#print(img.size)
if self.random_crop:
# GT IMAGE
# GT IMAGE -> 128*128*3 -----> 128*128*9
imgw, imgh = img.size
if imgh < self.image_shape[0] or imgw < self.image_shape[1]:
img = transforms.Resize(min(self.image_shape))(img)
img = transforms.RandomCrop(self.image_shape)(img)
# ORIGINAL IMAGE
imgw, imgh = orig_img.size
if imgh < self.image_shape[0] or imgw < self.image_shape[1]:
orig_img = transforms.Resize(min(self.image_shape))(orig_img)
orig_img = transforms.RandomCrop(self.image_shape)(orig_img)
else:
img = transforms.Resize(self.image_shape)(img)
img = transforms.RandomCrop(self.image_shape)(img)
orig_img = transforms.Resize(self.image_shape)(orig_img)
orig_img = transforms.RandomCrop(self.image_shape)(orig_img)
# img : (0 ~ 15), we have to convert it to 16 channel
#img = np.array(img, dtype=np.int32)
img = np.array(img, dtype=np.uint8)
#print("max : " + str(img.max()) + ", low : " + str(img.min()))
#img = transforms.ToTensor()(img).int() # turn the image to a tensor
img = torch.from_numpy(img).long()
#print(">max : " + str(img.max()) + ", low : " + str(img.min()))
#\img = normalize(img)
orig_img = transforms.ToTensor()(
orig_img) # turn the image to a tensor
orig_img = normalize(orig_img)
raw_name = self.samples[index].split('.')[0]
# =============================== OUTPUT IMAGE CAUTION SIZE!!!
#target = np.zeros([self.n_classes, 128, 128])
target = np.zeros([self.n_classes, 256, 256])
for c in range(self.n_classes):
target[c][img == c] = 1
target = torch.from_numpy(target).float()
#lbl_img = m.toimage(img, high=img.max(), low=img.min())
#print(lbl_img.shape)
# RETURN (NAME), GT, TARGET(ONE-HOT), ORIG
if self.return_name:
return raw_name, img, target, orig_img
else:
return img, target, orig_img
def generate(img, img_mask_path, model_path):
with torch.no_grad(): # enter no grad context
if img_mask_path and is_image_file(img_mask_path):
# Test a single masked image with a given mask
x = Image.fromarray(img)
mask = default_loader(img_mask_path)
x = transforms.Resize(config['image_shape'][:-1])(x)
x = transforms.CenterCrop(config['image_shape'][:-1])(x)
mask = transforms.Resize(config['image_shape'][:-1])(mask)
mask = transforms.CenterCrop(config['image_shape'][:-1])(mask)
x = transforms.ToTensor()(x)
mask = transforms.ToTensor()(mask)[0].unsqueeze(dim=0)
x = normalize(x)
x = x * (1. - mask)
x = x.unsqueeze(dim=0)
mask = mask.unsqueeze(dim=0)
elif img_mask_path:
raise TypeError("{} is not an image file.".format(img_mask_path))
else:
# Test a single ground-truth image with a random mask
#ground_truth = default_loader(img_path)
ground_truth = img
ground_truth = transforms.Resize(config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.CenterCrop(config['image_shape'][:-1])(ground_truth)
ground_truth = transforms.ToTensor()(ground_truth)
ground_truth = normalize(ground_truth)
ground_truth = ground_truth.unsqueeze(dim=0)
bboxes = random_bbox(config, batch_size=ground_truth.size(0))
x, mask = mask_image(ground_truth, bboxes, config)
# Set checkpoint path
if not model_path:
checkpoint_path = os.path.join('checkpoints',
config['dataset_name'],
config['mask_type'] + '_' + config['expname'])
else:
checkpoint_path = model_path
# Define the trainer
netG = Generator(config['netG'], cuda, device_ids)
# Resume weight
last_model_name = get_model_list(checkpoint_path, "gen", iteration=0)
if cuda:
netG.load_state_dict(torch.load(last_model_name))
else:
netG.load_state_dict(torch.load(last_model_name, map_location='cpu'))
model_iteration = int(last_model_name[-11:-3])
print("Resume from {} at iteration {}".format(checkpoint_path, model_iteration))
if cuda:
netG = nn.parallel.DataParallel(netG, device_ids=device_ids)
x = x.cuda()
mask = mask.cuda()
# Inference
x1, x2, offset_flow = netG(x, mask)
inpainted_result = x2 * mask + x * (1. - mask)
inpainted_result = from_torch_img_to_numpy(inpainted_result, 'output.png', padding=0, normalize=True)
return inpainted_result
def run(self, iterator, is_true_test=False):
if not self.net.has_model():
print("=> Please train your model first!!")
exit(1)
# build graph
self.net.model(self.X).optimizer(self.Y)
with tf.Session() as session:
# restore session
print("=> loading session from: {:s}".format(self.net.save_path))
self.net.saver.restore(session, self.net.save_path)
# file iterator
pre_output = []
for (index, flag, name, X, Y, Q, tamper_rate, shape, offset_x,
offset_y, width, height) in iterator():
(_X, _Y) = (utils.normalize(X),
utils.reformat(Y, self._conf.label_size))
(params, softmax, loss, predict,
accuracy) = session.run([
self.net.params_value, self.net.output, self.net.loss,
self.net.predict, self.net.accuracy
],
feed_dict={
self.X: _X,
self.Y: _Y
})
# tmp = [softmax(*label_size), predict, label, confidence]
tmp = np.zeros([256, 8], dtype=np.float32)
tmp[:, 0:5] = softmax
tmp[:, 5] = predict
tmp[:, 6] = Y
tmp[:, 7] = Q
tmp = np.reshape(tmp, (-1, 8))
pre_output.append(tmp)
# save result into csv file
if index == flag:
pre_output = np.array(pre_output, dtype=np.float32)
name = "{:s}_{:d}_{:d}".format(name, width, height)
df = pd.DataFrame(np.reshape(pre_output, (-1, 8)))
print(df.shape)
if not is_true_test:
csv_path = os.path.join(self.model_path, "pre-train",
"{:s}.csv".format(name))
else:
csv_path = os.path.join(self.model_path,
"pre-train-true",
"{:s}.csv".format(name))
print("=> save pretrain at csv_path={:s}".format(csv_path))
df.to_csv(csv_path)
pre_output = []
for i in range(len(predict)):
if predict[i] != Y[i]:
print(
"=> index={:d} p={:d} Y={:d} tamper_rate={:.5f} output={:s}"
.format(i, predict[i], Y[i], tamper_rate[i],
softmax[i]))
print(
"=> ({:d}, {:d}), loss={:.5f}, accuracy:{:.3f}% \n".format(
index, flag, loss, accuracy))
if len(params) > 0: print("\n\n")
print(
"=> pretrain csv format={f1,f2,f3...,predict,label,texture_quality}, f1,f2 is CNN confidence for each model\n"
)
g1 = (-x.grad)
g2 = (-y.grad)
disp2 = torch.cat((g1[None, ..., None], g2[None, ..., None]),
3).float()
disp2[disp2 != disp2] = 0
print('disp2')
vgrid = grid + disp2 / disp2.max() * 0.1
output = torch.nn.functional.grid_sample(X, vgrid)
Out = torch.tensor(output[0, 0, :, :].detach().numpy())
diff = -X[0, 0, :, :] + Out
X = normalize(X)
plt.figure(1)
plt.subplot(1, 3, 1)
plt.imshow(X[0, 0, :, :])
plt.title('input healthy')
plt.axis('off')
plt.subplot(1, 3, 2)
output = normalize(Out)
plt.subplot(1, 3, 2)
plt.imshow(Out)
plt.title('output diseased')
plt.axis('off')
plt.subplot(1, 3, 3)
plt.imshow((-diff), cmap='viridis')
plt.title('Differenz')
plt.axis('off')
