def export_all_contours(contours, data_path):
print('\nProcessing {:d} images and labels ...\n'.format(len(contours)))
images = np.zeros((len(contours), SIZE, SIZE, 1))
masks = np.zeros((len(contours), SIZE, SIZE, 1))
for idx, contour in enumerate(contours):
img, mask = read_contour(contour, data_path)
if img.shape[0] > SIZE:
img = center_crop(img, SIZE)
mask = center_crop(mask, SIZE)
images[idx] = img
masks[idx] = mask
return images, masks
def forward(self, x):
x = self.fully_connected(x) # 4096
x = x.reshape((-1, 256, 4, 4)) # 4x4x256
x = self.deconv(x) # 256x256x3
x = center_crop(x, self.deconv_output_size,
self.desired_output_size) # 227x227x3
return x
def forward(self, image, features):
cropped_img = center_crop(image, current_size=227,
desired_size=224) # 224x224x3
x1 = self.conv(cropped_img) # 1x1x256
x1 = torch.flatten(x1, 1) # 256
x2 = self.features_fc(features) # 512
x = torch.cat((x1, x2), dim=1) # 768
x = self.fc(x) # 1
return x
def load_eval_dataset(self):
(_, _), (x_test, self.y_test) = self.args.dataset.load_data()
image_size = x_test.shape[1]
x_test = np.reshape(x_test,[-1, image_size, image_size, 1])
x_test = x_test.astype('float32') / 255
x_eval = np.zeros([x_test.shape[0], *self.train_gen.input_shape])
for i in range(x_eval.shape[0]):
x_eval[i] = center_crop(x_test[i])
self.x_test = x_eval
def forward(self, x):
crop_h, crop_w = int(x.size()[-2]), int(x.size()[-1])
x = self.stages[0](x)
side = []
side_out = []
for i in range(1, len(self.stages)):
x = self.stages[i](x)
side_temp = self.side_prep[i - 1](x)
side.append(
center_crop(self.upscale[i - 1](side_temp), crop_h, crop_w))
side_out.append(
center_crop(
self.upscale_[i - 1](self.score_dsn[i - 1](side_temp)),
crop_h, crop_w))
out = torch.cat(side[:], dim=1)
out = self.fuse(out)
side_out.append(out)
return side_out
def img_process(self, imgs, frames_num): # imgs = 현재 지정된 프레임, frames_num = 네트워크에 들어갈 프레임 수
images = np.zeros((frames_num, 224, 224, 3))
orig_imgs = np.zeros_like(images)
for i in range(frames_num):
next_image = imgs[i]
next_image = np.uint8(next_image)
scaled_img = cv2.resize(next_image, (256, 256), interpolation=cv2.INTER_LINEAR) # resize to 256x256
cropped_img = center_crop(scaled_img) # center crop 224x224
final_img = cv2.cvtColor(cropped_img, cv2.COLOR_BGR2RGB)
images[i] = final_img # opencv와 달리 pytorch에선 rgb를 쓰기때문에 포맷이 변경된 이미지 사용.
orig_imgs[i] = cropped_img # opencv 는 BGR을 쓰기 때문에 포맷 유지
torch_imgs = torch.from_numpy(images.transpose(3, 0, 1, 2)) # 채널/ 프레임수/ 너비/ 높이
torch_imgs = torch_imgs.float() / 255.0
mean_3d = [124 / 255, 117 / 255, 104 / 255]
std_3d = [0.229, 0.224, 0.225]
for t, m, s in zip(torch_imgs, mean_3d, std_3d):
t.sub_(m).div_(s)
return np.expand_dims(orig_imgs, 0), torch_imgs.unsqueeze(0) # return opencv용 원본이미지, torch용 이미지
def layer_wise_relevance_propagation(conf):
img_dir = conf["paths"]["image_dir"]
res_dir = conf["paths"]["results_dir"]
image_height = conf["image"]["height"]
image_width = conf["image"]["width"]
lrp = RelevancePropagation(conf)
image_paths = list()
for (dirpath, dirnames, filenames) in os.walk(img_dir):
image_paths += [os.path.join(dirpath, file) for file in filenames]
for i, image_path in enumerate(image_paths):
print("Processing image {}".format(i+1))
image = center_crop(np.array(Image.open(image_path)), image_height, image_width)
relevance_map = lrp.run(image)
plot_relevance_map(image, relevance_map, res_dir, i)
def stylize_output(wct_model, content_img, styles):
if args.crop_size > 0:
styles = [center_crop(style) for style in styles]
if args.keep_colors:
styles = [preserve_colors_np(style, content_img) for style in styles]
# Run the frame through the style network
stylized = content_img
for _ in range(args.passes):
stylized = wct_model.predict(stylized, styles)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(
styles[0], (stylized.shape[0], stylized.shape[0]))
stylized = np.hstack([style_img_resized, stylized])
return stylized
def img_process(self, imgs, frames_num):
images = np.zeros((frames_num, 224, 224, 3))
orig_imgs = np.zeros_like(images)
for i in range(frames_num):
next_image = imgs[i]
next_image = np.uint8(next_image)
scaled_img = cv2.resize(
next_image, (256, 256),
interpolation=cv2.INTER_LINEAR) # resize to 256x256
cropped_img = center_crop(scaled_img) # center crop 224x224
final_img = cv2.cvtColor(cropped_img, cv2.COLOR_BGR2RGB)
images[i] = final_img
orig_imgs[i] = cropped_img
torch_imgs = torch.from_numpy(images.transpose(3, 0, 1, 2))
torch_imgs = torch_imgs.float() / 255.0
mean_3d = [124 / 255, 117 / 255, 104 / 255]
std_3d = [0.229, 0.224, 0.225]
for t, m, s in zip(torch_imgs, mean_3d, std_3d):
t.sub_(m).div_(s)
return np.expand_dims(orig_imgs, 0), torch_imgs.unsqueeze(0)
def __getitem__(self, idx):
self.current_item_path = self.inputs[idx]
input_img = correct_dims(nib.load(self.inputs[idx]).get_data())
label_img = gen_mask(self.labels[idx])
# Resize to input image and label to size (self.size x self.size x self.size)
if self.sampling_mode == "resize":
ex, label = resize_img(input_img, label_img, self.size)
# Constant center-crop sample of size (self.size x self.size x self.size)
elif self.sampling_mode == 'center':
ex, label = center_crop(input_img, label_img, self.size)
# Find centers of lesion masks and crop image to include them
# to measure consistent validation performance with small crops
elif self.sampling_mode == "center_val":
ex, label = find_and_crop_lesions(input_img, label_img, self.size,
self.deterministic)
# Randomly crop sample of size (self.size x self.size x self.size)
elif self.sampling_mode == "random":
ex, label = random_crop(input_img, label_img, self.size)
else:
print("Invalid sampling mode.")
exit()
ex = np.divide(ex, 255.0)
label = np.array([(label > 0).astype(int)]).squeeze()
# (experimental) APPLY RANDOM FLIPPING ALONG EACH AXIS
if not self.deterministic:
for i in range(3):
if random() > 0.5:
ex = np.flip(ex, i)
label = np.flip(label, i)
inputs = torch.from_numpy(ex.copy()).type(
torch.FloatTensor).unsqueeze(0)
labels = torch.from_numpy(label.copy()).type(
torch.FloatTensor).unsqueeze(0)
return inputs, labels
def pre_process_input_data(data, sample=None):
'''Apply transformation fuctions (normalization,
sampling frames and center cropping) on a video array.
Args:
data--> a 4D array of video frames
sample--> sample a fixed number of frames from the video (int)
Returns:
data --> a tranformed 5D data array with batch size
'''
data = normalize(np.array(data))
if sample is not None:
data, _ = get_random_frames(data, label=0, num_frames=sample)
data, _ = center_crop(data,
target_size=(config.get('HEIGHT'),
config.get('WIDTH')))
data = np.expand_dims(data, axis=0)
return data.astype(np.float32)
def evaluate(model_path, file_dir, meta_file):
'''Evaluates a trained model on the test samples and outputs the
overall and average accuracy values.
Args:
model_path--> path to the keras saved model
file_dir --> path to the video files
meta_file --> meta file containing the test video file names and labels
Returns:
report--> returns a text report showing main classification metrics using
scikit-learn classification report method
'''
loaded_model = K.models.load_model(model_path, custom_objects={'B': B})
test_data, test_label = get_file_list(file_dir,
meta_file,
file_type='.mp4')
test_prediction = []
for x, y in zip(test_data, test_label):
video = read_video(x)
video = normalize(video)
video, _ = center_crop(video,
target_size=(config_data.get('HEIGHT'),
config_data.get('WIDTH')))
prob = get_pridiction(loaded_model, video)
test_prediction.append(prob[0])
label_map = read_json(LABEL_MAP_FILE)
vocabulary = [
label_map.get(str(i)) for i in range(config_data.get('NUM_CLASSES'))
]
report = classification_report(test_label,
test_prediction,
target_names=vocabulary,
zero_division=1)
return report
def main():
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Create needed dirs
in_dir = os.path.join(args.tmp_dir, 'input')
out_dir = os.path.join(args.tmp_dir, 'sytlized')
if not os.path.exists(in_dir):
os.makedirs(in_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
if os.path.isdir(args.in_path):
in_path = get_files(args.in_path)
else: # Single image file
in_path = [args.in_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
else: # Single image file
style_files = [args.style_path]
print(style_files)
import time
# time.sleep(999)
in_args = [
'ffmpeg',
'-i', args.in_path,
'%s/frame_%%d.png' % in_dir
]
subprocess.call(" ".join(in_args), shell=True)
base_names = os.listdir(in_dir)
in_files = [os.path.join(in_dir, x) for x in base_names]
out_files = [os.path.join(out_dir, x) for x in base_names]
s = time.time()
for content_fullpath in in_path:
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(content_prefix)
try:
for style_fullpath in style_files:
style_img = get_img(style_fullpath)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(style_prefix)
# print("ARRAY: ", style_img)
out_v = os.path.join(args.out_path, '{}_{}{}'.format(content_prefix, style_prefix, content_ext))
print("OUT:",out_v)
if os.path.isfile(out_v):
print("SKIP" , out_v)
continue
for in_f, out_f in zip(in_files, out_files):
print('{} -> {}'.format(in_f, out_f))
content_img = get_img(in_f)
if args.keep_colors:
style_rgb = preserve_colors_np(style_img, content_img)
else:
style_rgb = style_img
stylized = wct_model.predict(content_img, style_rgb, args.alpha, args.swap5, args.ss_alpha)
if args.passes > 1:
for _ in range(args.passes-1):
stylized = wct_model.predict(stylized, style_rgb, args.alpha)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(style_rgb, (stylized.shape[0], stylized.shape[0]))
stylized = np.hstack([style_img_resized, stylized])
save_img(out_f, stylized)
fr = 30
out_args = [
'ffmpeg',
'-i', '%s/frame_%%d.png' % out_dir,
'-f', 'mp4',
'-q:v', '0',
'-vcodec', 'mpeg4',
'-r', str(fr),
'"' + out_v + '"'
]
print(out_args)
subprocess.call(" ".join(out_args), shell=True)
print('Video at: %s' % out_v)
if args.keep_tmp is True or len(style_files) > 1:
continue
else:
shutil.rmtree(args.tmp_dir)
print('Processed in:',(time.time() - s))
print('Processed in:',(time.time() - s))
except Exception as e:
print("EXCEPTION: ",e)
def main():
start = time.time()
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Get content & style full paths
if os.path.isdir(args.content_path):
content_files = get_files(args.content_path)
else: # Single image file
content_files = [args.content_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
if args.random > 0:
style_files = np.random.choice(style_files, args.random)
else: # Single image file
style_files = [args.style_path]
os.makedirs(args.out_path, exist_ok=True)
count = 0
### Apply each style to each content image
for content_fullpath in content_files:
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(content_prefix) # Extract filename prefix without ext
content_img = get_img(content_fullpath)
if args.content_size > 0:
content_img = resize_to(content_img, args.content_size)
for style_fullpath in style_files:
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(style_prefix) # Extract filename prefix without ext
# style_img = get_img_crop(style_fullpath, resize=args.style_size, crop=args.crop_size)
# style_img = resize_to(get_img(style_fullpath), content_img.shape[0])
style_img = get_img(style_fullpath)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
if args.keep_colors:
style_img = preserve_colors_np(style_img, content_img)
# if args.noise: # Generate textures from noise instead of images
# frame_resize = np.random.randint(0, 256, frame_resize.shape, np.uint8)
# frame_resize = gaussian_filter(frame_resize, sigma=0.5)
# Run the frame through the style network
stylized_rgb = wct_model.predict(content_img, style_img, args.alpha, args.swap5, args.ss_alpha, args.adain)
if args.passes > 1:
for _ in range(args.passes-1):
stylized_rgb = wct_model.predict(stylized_rgb, style_img, args.alpha, args.swap5, args.ss_alpha, args.adain)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(style_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
# margin = np.ones((style_img_resized.shape[0], 10, 3)) * 255
stylized_rgb = np.hstack([style_img_resized, stylized_rgb])
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
out_f = os.path.join(args.out_path, '{}_{}{}'.format(content_prefix, style_prefix, content_ext))
# out_f = f'{content_prefix}_{style_prefix}.{content_ext}'
save_img(out_f, stylized_rgb)
count += 1
print("{}: Wrote stylized output image to {}".format(count, out_f))
print("Finished stylizing {} outputs in {}s".format(count, time.time() - start))
def __getitem__(self, index):
subj_file = self.datafiles[index]
with h5py.File(self.dirname + subj_file, 'r') as fdset:
h5data = fdset['kspace']
sh = h5data.shape
sliceindex = np.random.randint(0, sh[0])
ksp_sli = h5data[sliceindex]
C, H, W = ksp_sli.shape
# Rotate ksp
img_sli = np.fft.ifftshift(np.fft.ifft2(np.fft.ifftshift(ksp_sli,
axes=(1, 2)),
axes=(1, 2)),
axes=(1, 2))
ksp_sli = np.fft.fftshift(np.fft.fft2(img_sli, axes=(1, 2)),
axes=(1, 2))
if self.rss == 'norm_acl':
#mean, std = mean_std_aclines(ksp_sli, 10)
i_min, i_max = min_max_aclines(ksp_sli, 10)
img_sli_singlec = np.sqrt(
np.sum(np.square(np.abs(img_sli)), axis=0))
#norm_img_sli_singlec = (img_sli_singlec-mean)/std
#print(norm_img_sli_singlec.max(), norm_img_sli_singlec.mean(), norm_img_sli_singlec.min())
norm_img_sli_singlec = (img_sli_singlec - i_min) / (i_max - i_min)
elif not self.rss:
try:
est_coilmap = np.load(self.coil_path + subj_file +
str(sliceindex) + '.npy')
except:
print('CANT FIND:',
self.coil_path + subj_file + str(sliceindex),
' Creating new...')
est_coilmap_gpu = mr.app.EspiritCalib(ksp_sli,
calib_width=W,
thresh=0.02,
kernel_width=6,
crop=0.01,
max_iter=100,
show_pbar=False,
device=-1).run()
est_coilmap = np.fft.fftshift(est_coilmap_gpu, axes=(1, 2))
np.save(self.coil_path + subj_file + str(sliceindex),
est_coilmap)
if np.isnan(np.min(est_coilmap)):
print('COILMAP Contains nan. Create new...')
est_coilmap_gpu = mr.app.EspiritCalib(ksp_sli,
calib_width=W,
thresh=0.02,
kernel_width=6,
crop=0.01,
max_iter=100,
show_pbar=False,
device=-1).run()
est_coilmap = np.fft.fftshift(est_coilmap_gpu, axes=(1, 2))
np.save(self.coil_path + subj_file + str(sliceindex),
est_coilmap)
if np.isnan(np.min(est_coilmap)):
print('COILMAP still contains nan...')
# Normalise
img_sli_singlec = np.sum(
np.conjugate(est_coilmap) * img_sli, axis=0) / (np.sum(
np.conjugate(est_coilmap) * est_coilmap, axis=0) + 1e-6)
norm_fac = 1 / (np.percentile(
np.abs(img_sli_singlec).flatten(), 80))
norm_img_sli_singlec = norm_fac * np.abs(img_sli_singlec) * np.exp(
1j * np.angle(img_sli_singlec)) #
else:
img_sli_singlec = np.sqrt(
np.sum(np.square(np.abs(img_sli)), axis=0))
norm_fac = 1 / (np.percentile(
np.abs(img_sli_singlec).flatten(), 80))
norm_img_sli_singlec = norm_fac * np.abs(img_sli_singlec)
if self.crop:
norm_img_sli_singlec = center_crop(norm_img_sli_singlec) + 1j * 0
if self.noise > 0:
norm_img_sli_singlec = (
np.abs(norm_img_sli_singlec) +
np.random.normal(loc=0,
scale=1 / self.noise,
size=norm_img_sli_singlec.shape)) * np.exp(
1j * np.angle(norm_img_sli_singlec))
if np.isnan(np.min(norm_img_sli_singlec)):
norm_img_sli_singlec = np.zeros_like(norm_img_sli_singlec)
# Create Patches
subj_tens = torch.from_numpy(norm_img_sli_singlec.real).type(
torch.complex64) + 1j * torch.from_numpy(
norm_img_sli_singlec.imag).type(torch.complex64)
patches = subj_tens.unfold(0, self.patchsize, self.patchsize).unfold(
1, self.patchsize, self.patchsize)
patches = torch.cat(
patches.unbind()) # Patches is now [Num_patches, 28,28]
return patches
def main():
start = time.time()
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Get content & style full paths
if os.path.isdir(args.content_path):
content_files = get_files(args.content_path)
else: # Single image file
content_files = [args.content_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
if args.random > 0:
style_files = np.random.choice(style_files, args.random)
else: # Single image file
style_files = [args.style_path]
os.makedirs(args.out_path, exist_ok=True)
count = 0
# Apply each style to each content image
for content_fullpath in content_files:
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(
content_prefix) # Extract filename prefix without ext
content_img = get_img(content_fullpath)
if args.content_size > 0:
content_img = resize_to(content_img, args.content_size)
for style_fullpath in style_files:
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(
style_prefix) # Extract filename prefix without ext
# style_img = get_img_crop(style_fullpath, resize=args.style_size, crop=args.crop_size)
# style_img = resize_to(get_img(style_fullpath), content_img.shape[0])
style_img = get_img(style_fullpath)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
if args.keep_colors:
style_img = preserve_colors_np(style_img, content_img)
# if args.noise: # Generate textures from noise instead of images
# frame_resize = np.random.randint(0, 256, frame_resize.shape, np.uint8)
# frame_resize = gaussian_filter(frame_resize, sigma=0.5)
# Run the frame through the style network
stylized_rgb = wct_model.predict(content_img, style_img,
args.alpha, args.swap5,
args.ss_alpha, args.adain)
if args.passes > 1:
for _ in range(args.passes - 1):
stylized_rgb = wct_model.predict(stylized_rgb, style_img,
args.alpha, args.swap5,
args.ss_alpha, args.adain)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(
style_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
# margin = np.ones((style_img_resized.shape[0], 10, 3)) * 255
stylized_rgb = np.hstack([style_img_resized, stylized_rgb])
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
out_f = os.path.join(
args.out_path, '{}_{}{}'.format(content_prefix, style_prefix,
content_ext))
# out_f = f'{content_prefix}_{style_prefix}.{content_ext}'
save_img(out_f, stylized_rgb)
count += 1
print("{}: Wrote stylized output image to {}".format(count, out_f))
print("Finished stylizing {} outputs in {}s".format(
count,
time.time() - start))
def freeze_graph_test(pb_file, content_path, style_path):
'''
:param pb_path:pb文件的路径
:param image_path:测试图片的路径
:return:
'''
content_prefix, content_ext = os.path.splitext(content_path)
content_prefix = os.path.basename(
content_prefix) # Extract filename prefix without ext
style_prefix, _ = os.path.splitext(style_path)
style_prefix = os.path.basename(
style_prefix) # Extract filename prefix without ext
# 读取测试图片content_img和style_img
content_img = get_img(content_path)
if args.content_size > 0:
content_img = resize_to(content_img, args.content_size)
style_img = get_img(style_path)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
# if args.keep_colors:
# style_img = preserve_colors_np(style_img, content_img)
content_img = preprocess(content_img)
style_img = preprocess(style_img)
# 打印检查点所有的变量
chkp.print_tensors_in_checkpoint_file("models/inference/model.ckpt",
tensor_name='',
all_tensors=False)
# for i in range(5):
# print("=" * 10, i + 1)
# chkp.print_tensors_in_checkpoint_file("models/relu" + str(i+1) + "_1/model.ckpt-15002", tensor_name='', all_tensors=True)
with tf.Graph().as_default():
output_graph_def = tf.GraphDef()
# with open(pb_file, "rb") as f:
# x = f.read()
# # print(x)
# output_graph_def.ParseFromString(x)
# output_graph_def.ParseFromString(f.read())
# tf.import_graph_def(output_graph_def, name="")
# for i, n in enumerate(output_graph_def.node):
# print("Name of the node - %s" % n.name)
# print(n)
with tf.Session() as sess:
# sess.run(tf.global_variables_initializer())
# tf.saved_model.loader.load(sess, [tag_constants.TRAINING], export_dir)
# 定义输入的张量名称,对应网络结构的输入张量
content_input_tensor = sess.graph.get_tensor_by_name(
"encoder_decoder_relu5_1/content_encoder_relu5_1/content_imgs:0"
)
style_input_tensor = sess.graph.get_tensor_by_name("style_img:0")
# 定义输出的张量名称
output_tensor_name = sess.graph.get_tensor_by_name(
"encoder_decoder_relu5_1/decoder_relu5_1/decoder_model_relu5_1/relu5_1_16/relu5_1_16/BiasAdd:0"
)
# 测试读出来的模型是否正确，注意这里传入的是输出和输入节点的tensor的名字，不是操作节点的名字
stylized_rgb = sess.run(output_tensor_name,
feed_dict={
content_input_tensor: content_img,
style_input_tensor: style_img
})
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(
style_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
stylized_rgb = np.hstack([style_img_resized, stylized_rgb])
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
out_f = os.path.join(
args.out_path, '{}_{}{}'.format(content_prefix, style_prefix,
content_ext))
save_img(out_f, stylized_rgb)
import glob
import os
import utils
import scipy.misc
PATH_IN = '../celebA'
PATH_OUT = '../celeba_norm'
os.makedirs(PATH_OUT, exist_ok=True)
data_files = glob.glob(os.path.join(PATH_IN, "*.jpg"))
for pin in data_files:
pout = pin.replace('celebA', 'celeba_norm')
f = scipy.misc.imread(pin)
f = utils.center_crop(f, 108, 64)
scipy.misc.imsave(pout, f)
print(pout)
print('Done')
def loop(data_info, config, split_stack=True, test_time=True):
up_thres, low_thres = config.up_thres, config.low_thres
all_ims = []
for info in data_info:
depth_in_path, depth_ref_path, color_path, mask_path = info
name = os.path.basename(depth_in_path)
raw = Image.open(depth_in_path)
gt = Image.open(depth_ref_path)
rgb = Image.open(color_path).convert('L').resize(raw.size)
mask = Image.open(mask_path)
assert raw.size == gt.size, 'gt size not match raw size!'
# Do center crop here
if not split_stack:
if config.image_size < min(raw.size):
raw, gt, rgb, mask = utils.center_crop(raw, gt, rgb, mask, config.image_size)
elif config.image_size >= max(raw.size):
raw, gt, rgb, mask = utils.center_pad(raw, gt, rgb, mask, config.image_size)
else:
raise NotImplementedError('invalid config.image_size.')
mask = np.array(mask, dtype=np.float32) / 255.0
rgb = np.array(rgb, dtype=np.float32) / (127.0 - 1.0) * mask
thres_range = (up_thres - low_thres) / 2.0
raw = np.clip(np.array(raw, dtype=np.float32), low_thres, up_thres)
gt = np.clip(np.array(gt, dtype=np.float32), low_thres, up_thres)
if config.dnnet == "bilateral":
raw = cv2.bilateralFilter(raw, 9, 75, 75)
raw = (raw - low_thres) / thres_range - 1.0
gt = (gt - low_thres) / thres_range - 1.0
if split_stack:
raw_arr, H, W = utils.split_patch(raw, config.image_size)
gt_arr, _, _ = utils.split_patch(gt, config.image_size)
rgb_arr, _, _ = utils.split_patch(rgb, config.image_size)
all_ims.append((name, raw_arr, gt_arr, rgb_arr))
else:
all_ims.append((name, raw, gt, rgb, mask))
ckpt_history = list()
if test_time and not split_stack:
path = tf.train.latest_checkpoint(config.checkpoint_dir)
print("Load checkpoint: {}".format(path))
params = build_model(config.image_size, config.image_size, config)
sess = tf.Session()
load_from_checkpoint(sess, path)
tt_time = 0.0
history_len = 3
t_history = np.zeros(history_len, dtype=np.float32)
for i, (name, raw, _, rgb, mask) in enumerate(all_ims):
t_elapsed = loop_body_patch_time(sess, name, params, raw, rgb, mask, config)
t_history[i % history_len] = t_elapsed
tt_time += t_elapsed
avg_time = 1000 * tt_time / (i + 1) # ms
mv_avg_time = 1000 * np.mean(t_history)
print('iter {} | tt_time: {:.4f}s; avg_time: {:.2f}; mv_avg_time: {:.2f}'.format(i+1, tt_time, avg_time, mv_avg_time))
tf.reset_default_graph()
else:
while True:
# Wait until new checkpoint exist when training phase is not finished.
path = wait_for_new_checkpoint(config.checkpoint_dir, ckpt_history)
print("Loading from checkpoint: {}".format(path))
if split_stack:
print('evaluating {} imgs'.format(len(all_ims)))
for i, (name, raw_arr, gt_arr, rgb_arr) in enumerate(all_ims):
loop_body_whole(i, path, raw_arr, gt_arr, rgb_arr, H, W, config)
else:
for i, (name, raw, gt, rgb, mask) in enumerate(all_ims):
loop_body_patch(i, path, raw, gt, rgb, mask, config)
if not config.loop: break
# image sequence numbering
frame_seq = change_fps(5, frames, length)
print('Total New Frame:', len(frame_seq))
frame_n = 0
i = 0
while (cap.isOpened()):
# get frame-by-frame
hasVideo, frame = cap.read()
if hasVideo == True and frame_n == round(frame_seq[i]):
# resize
#frame = cv2.resize(frame, (1382, 512), interpolation=cv2.INTER_AREA)
# crop image
frame = center_crop(frame, (1382, 512))
# display video
cv2.imshow('Frame', frame)
# write frame to image
cv2.imwrite(f'sequence_04/{i:06d}.png', frame)
# presss q to break
if cv2.waitKey(25) & 0xFF == ord('q'):
break
i = i + 1
frame_n = frame_n + 1
if frame_n == frames:
break
# release video
cap.release()
def main():
# start = time.time()
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Get content & style full paths
if os.path.isdir(args.content_path):
content_files = get_files(args.content_path)
else: # Single image file
content_files = [args.content_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
if args.random > 0:
style_files = np.random.choice(style_files, args.random)
else: # Single image file
style_files = [args.style_path]
os.makedirs(args.out_path, exist_ok=True)
thetotal = len(content_files) * len(style_files)
count = 1
### Apply each style to each content image
for content_fullpath in content_files:
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(
content_prefix) # Extract filename prefix without ext
content_img = get_img(content_fullpath)
if args.content_size > 0:
content_img = resize_to(content_img, args.content_size)
for style_fullpath in style_files:
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(
style_prefix) # Extract filename prefix without ext
if args.mask:
mask_prefix_, _ = os.path.splitext(args.mask)
mask_prefix = os.path.basename(mask_prefix_)
if args.keep_colors:
style_prefix = "KPT_" + style_prefix
if args.concat:
style_prefix = "CON_" + style_prefix
if args.adain:
style_prefix = "ADA_" + style_prefix
if args.swap5:
style_prefix = "SWP_" + style_prefix
if args.mask:
style_prefix = "MSK_" + mask_prefix + '_' + style_prefix
if args.remaster:
style_prefix = style_prefix + '_REMASTERED'
out_f = os.path.join(
args.out_path, '{}_{}{}'.format(content_prefix, style_prefix,
content_ext))
if os.path.isfile(out_f):
print("SKIP", out_f)
count += 1
continue
# style_img = get_img_crop(style_fullpath, resize=args.style_size, crop=args.crop_size)
# style_img = resize_to(get_img(style_fullpath), content_img.shape[0])
style_img = get_img(style_fullpath)
if style_img == ("IMAGE IS BROKEN"):
continue
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
if args.keep_colors:
style_img = preserve_colors_np(style_img, content_img)
# if args.noise: # Generate textures from noise instead of images
# frame_resize = np.random.randint(0, 256, frame_resize.shape, np.uint8)
# frame_resize = gaussian_filter(frame_resize, sigma=0.5)
# Run the frame through the style network
stylized_rgb = wct_model.predict(content_img, style_img,
args.alpha, args.swap5,
args.ss_alpha, args.adain)
if args.passes > 1:
for _ in range(args.passes - 1):
stylized_rgb = wct_model.predict(stylized_rgb, style_img,
args.alpha, args.swap5,
args.ss_alpha, args.adain)
if args.mask:
import cv2
cv2.imwrite('./tmp.png', stylized_rgb)
stylized_rgb = cv2.imread('./tmp.png')
mask = cv2.imread(args.mask, cv2.IMREAD_GRAYSCALE)
# from scipy.misc import bytescale
# mask = bytescale(mask)
# mask = scipy.ndimage.imread(args.mask,flatten=True,mode='L')
height, width = stylized_rgb.shape[:2]
# print(height, width)
# Resize the mask to fit the image.
mask = scipy.misc.imresize(mask, (height, width),
interp='bilinear')
stylized_rgb = cv2.bitwise_and(stylized_rgb,
stylized_rgb,
mask=mask)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
# style_prefix = style_prefix + "CON_"
# content_img_resized = scipy.misc.imresize(content_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
style_img_resized = scipy.misc.imresize(
style_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
# margin = np.ones((style_img_resized.shape[0], 10, 3)) * 255
stylized_rgb = np.hstack([style_img_resized, stylized_rgb])
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
# out_f = f'{content_prefix}_{style_prefix}.{content_ext}'
out_f = os.path.join(
args.out_path, '{}_{}{}'.format(content_prefix, style_prefix,
content_ext))
if args.remaster:
# outf = os.path.join(args.out_path, '{}_{}_REMASTERED{}'.format(content_prefix, style_prefix, content_ext))
stylized_rgb = remaster_pic(stylized_rgb)
save_img(out_f, stylized_rgb)
totalfiles = len([
name for name in os.listdir(args.out_path)
if os.path.isfile(os.path.join(args.out_path, name))
])
# percent = math.floor(float(totalfiles/thetotal))
print("{}/{} TOTAL FILES".format(count, thetotal))
count += 1
print("{}: Wrote stylized output image to {}".format(count, out_f))
def main():
start = time.time()
session_conf = tf.ConfigProto(
allow_soft_placement=args.allow_soft_placement,
log_device_placement=args.log_device_placement)
session_conf.gpu_options.per_process_gpu_memory_fraction = args.gpu_fraction
with tf.Graph().as_default():
# with tf.Session(config=session_conf) as sess:
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
with wct_model.sess as sess:
# 训练的时候不需要style_img，所以在inference的时候重新保存一次checkpoint
saver = tf.train.Saver()
log_path = args.log_path if args.log_path is not None else os.path.join(
args.checkpoint, 'log')
summary_writer = tf.summary.FileWriter(log_path, sess.graph)
# 部分node没有保存在checkpoint中，需要重新初始化
# sess.run(tf.global_variables_initializer())
# Get content & style full paths
if os.path.isdir(args.content_path):
content_files = get_files(args.content_path)
else: # Single image file
content_files = [args.content_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
if args.random > 0:
style_files = np.random.choice(style_files, args.random)
else: # Single image file
style_files = [args.style_path]
os.makedirs(args.out_path, exist_ok=True)
count = 0
# Apply each style to each content image
for content_fullpath in content_files:
content_prefix, content_ext = os.path.splitext(
content_fullpath)
content_prefix = os.path.basename(
content_prefix) # Extract filename prefix without ext
content_img = get_img(content_fullpath)
if args.content_size > 0:
content_img = resize_to(content_img, args.content_size)
for style_fullpath in style_files:
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(
style_prefix) # Extract filename prefix without ext
# style_img = get_img_crop(style_fullpath, resize=args.style_size, crop=args.crop_size)
# style_img = resize_to(get_img(style_fullpath), content_img.shape[0])
style_img = get_img(style_fullpath)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
if args.keep_colors:
style_img = preserve_colors_np(style_img, content_img)
# if args.noise: # Generate textures from noise instead of images
# frame_resize = np.random.randint(0, 256, frame_resize.shape, np.uint8)
# frame_resize = gaussian_filter(frame_resize, sigma=0.5)
# Run the frame through the style network
stylized_rgb = wct_model.predict(content_img, style_img,
args.alpha, args.swap5,
args.ss_alpha, args.adain)
if args.passes > 1:
for _ in range(args.passes - 1):
stylized_rgb = wct_model.predict(
stylized_rgb, style_img, args.alpha,
args.swap5, args.ss_alpha, args.adain)
save_path = saver.save(
sess, os.path.join(args.checkpoint, 'model.ckpt'))
print("Model saved in file: %s" % save_path)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(
style_img,
(stylized_rgb.shape[0], stylized_rgb.shape[0]))
# margin = np.ones((style_img_resized.shape[0], 10, 3)) * 255
stylized_rgb = np.hstack(
[style_img_resized, stylized_rgb])
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
out_f = os.path.join(
args.out_path,
'{}_{}{}'.format(content_prefix, style_prefix,
content_ext))
# out_f = f'{content_prefix}_{style_prefix}.{content_ext}'
# print(stylized_rgb, stylized_rgb.shape, type(stylized_rgb))
# print(out_f)
save_img(out_f, stylized_rgb)
count += 1
print("{}: Wrote stylized output image to {} at {}".format(
count, out_f, time.time()))
print("breaking...")
break
break
print("Finished stylizing {} outputs in {}s".format(
count,
time.time() - start))
def main():
start = time.time()
# Load the WCT model
wct_model = WCT(input_checkpoint=args.input_checkpoint,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
print("model construct end !")
# Get content & style full paths
if os.path.isdir(args.content_path):
content_files = get_files(args.content_path)
else: # Single image file
content_files = [args.content_path]
content_seg_files = list(map(lambda x: x.replace("2017", "2017_seg").replace("jpg", "png"), content_files))
assert reduce(lambda a, b : a + b, map(lambda x: int(os.path.exists(x)),content_seg_files + content_files)) > 0
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
if args.random > 0:
style_files = np.random.choice(style_files, args.random)
else: # Single image file
style_files = [args.style_path]
style_seg_files = list(map(lambda x: x.replace("2017", "2017_seg").replace("jpg", "png"), style_files))
assert reduce(lambda a, b : a + b, map(lambda x: int(os.path.exists(x)),style_seg_files + style_files)) > 0
os.makedirs(args.out_path, exist_ok=True)
count = 0
### Apply each style to each content image
for i in range(len(content_files)):
content_fullpath = content_files[i]
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(content_prefix) # Extract filename prefix without ext
content_img = get_img(content_fullpath)
if args.content_size > 0:
content_img = np.asarray(Image.fromarray(content_img.astype(np.uint8)).resize((args.content_size, args.content_size))).astype(np.float32)
content_seg = get_img_ori(content_seg_files[i])
if args.content_size > 0:
content_seg = np.asarray(Image.fromarray(content_seg.astype(np.uint8)).resize((args.content_size, args.content_size))).astype(np.float32)
content_seg = image_to_pixel_map(content_seg)[...,np.newaxis]
content_img = np.concatenate([content_img, content_seg], axis=-1)
for i in range(len(style_files)):
style_fullpath = style_files[i]
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(style_prefix) # Extract filename prefix without ext
style_img = get_img(style_fullpath)
style_seg = get_img_ori(style_seg_files[i])
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
style_seg = resize_to(style_seg, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
style_seg = center_crop(style_seg, args.crop_size)
style_seg = image_to_pixel_map(style_seg)[...,np.newaxis]
style_img = np.concatenate([style_img, style_seg], axis=-1)
assert not args.keep_colors
if args.keep_colors:
style_img = preserve_colors_np(style_img, content_img)
# if args.noise: # Generate textures from noise instead of images
# frame_resize = np.random.randint(0, 256, frame_resize.shape, np.uint8)
# frame_resize = gaussian_filter(frame_resize, sigma=0.5)
# Run the frame through the style network
stylized_rgb = wct_model.predict(content_img, style_img, args.alpha, args.swap5, args.ss_alpha, args.adain)
if args.passes > 1:
for _ in range(args.passes-1):
stylized_rgb = np.concatenate([stylized_rgb ,content_seg], axis=-1)
stylized_rgb = wct_model.predict(stylized_rgb, style_img, args.alpha, args.swap5, args.ss_alpha, args.adain)
# Stitch the style + stylized output together, but only if there's one style image
assert not args.concat
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(style_img, (stylized_rgb.shape[0], stylized_rgb.shape[0]))
# margin = np.ones((style_img_resized.shape[0], 10, 3)) * 255
stylized_rgb = np.hstack([style_img_resized, stylized_rgb])
####+++++++++++++++++++++++++++++++++++
# Format for out filename: {out_path}/{content_prefix}_{style_prefix}.{content_ext}
out_f = os.path.join(args.out_path, '{}_{}{}'.format(content_prefix, style_prefix, content_ext))
save_img(out_f, stylized_rgb)
count += 1
print("{}: Wrote stylized output image to {}".format(count, out_f))
print("Finished stylizing {} outputs in {}s".format(count, time.time() - start))
'/volume/annahung-project/image_generation/draw-the-music/annadraw/dataset/wikiart/*/*'
)
print('dataset:', len(dataset))
names = verify(dataset)
folds = 5
random.shuffle(names)
images = {}
count = len(names)
print("Count: %d" % count)
count_per_fold = count // folds
i = 0
im = 0
for imgfile in tqdm(names):
image = center_crop(imageio.imread(imgfile))
images[imgfile] = image
im += 1
if im == count_per_fold:
output = open(dataset_name + '_data_fold_%d.pkl' % i, 'wb')
pickle.dump(list(images.values()), output)
output.close()
i += 1
im = 0
images.clear()
def run_inference(subj, R, mode, k, num_sampels, num_bootsamles, batch_size,
num_iter, step_size, phase_step, complex_rec, use_momentum,
log, device):
# Some inits of paths... Edit these
vae_model_name = 'T2-20210415-111101/450.pth'
vae_path = '/cluster/scratch/jonatank/logs/ddp/vae/'
data_path = '/cluster/work/cvl/jonatank/fastMRI_T2/validation/'
log_path = '/cluster/scratch/jonatank/logs/ddp/restore/pytorch/'
rss = True
# Load pretrained VAE
path = vae_path + vae_model_name
vae = torch.load(path, map_location=torch.device(device))
vae.eval()
# Data loader setup
subj_dataset = Subject(subj, data_path, R, rss=rss)
subj_loader = data.DataLoader(subj_dataset,
batch_size=1,
shuffle=False,
num_workers=0)
# Time model and init resulting matrices
start_time = time.perf_counter()
rec_subj = np.zeros((len(subj_loader), 320, 320))
gt_subj = np.zeros((len(subj_loader), 320, 320))
# Set basic parameters
print('Subj: ', subj, ' R: ', R, ' mode: ', mode, ' k: ', k,
' num_sampels: ', num_sampels, ' num_bootsamles: ', num_bootsamles,
' batch_size: ', batch_size, ' num_iter: ', num_iter, ' step_size: ',
step_size, ' phase_step: ', phase_step)
# Log
log_path = log_path + 'R' + str(R) + '_mode' + str(
k) + mode + '_reg2lmb0.01_' + datetime.now().strftime("%Y%m%d-%H%M%S")
if log:
import wandb
wandb.login()
wandb.init(project='JDDP' + '_T2',
name=vae_model_name,
config={
"num_iter": num_iter,
"step_size": step_size,
"phase_step": phase_step,
"mode": mode,
'R': R,
'K': k,
'use_momentum': use_momentum
})
#wandb.watch(vae)
else:
wandb = False
print("num_iter", num_iter, " step_size ", step_size, " phase_step ",
phase_step, " mode ", mode, ' R ', R, ' K ', k, 'use_momentum',
use_momentum)
for batch in tqdm(subj_loader, desc="Running inference"):
ksp, coilmaps, rss, norm_fact, num_sli = batch
rec_sli = vaerecon(ksp[0],
coilmaps[0],
mode,
vae,
rss[0],
log_path,
device,
writer=wandb,
norm=norm_fact.item(),
nsampl=num_sampels,
boot_samples=num_bootsamles,
k=k,
patchsize=28,
parfact=batch_size,
num_iter=num_iter,
stepsize=step_size,
lmb=phase_step,
use_momentum=use_momentum)
rec_subj[num_sli] = np.abs(center_crop(rec_sli.detach().cpu().numpy()))
gt_subj[num_sli] = np.abs(center_crop(rss[0]))
rmse_sli = nmse(rec_subj[num_sli], gt_subj[num_sli])
ssim_sli = ssim(rec_subj[num_sli], gt_subj[num_sli])
psnr_sli = psnr(rec_subj[num_sli], gt_subj[num_sli])
print('Slice: ', num_sli.item(), ' RMSE: ', str(rmse_sli), ' SSIM: ',
str(ssim_sli), ' PSNR: ', str(psnr_sli))
end_time = time.perf_counter()
print(f"Elapsed time for {str(num_sli)} slices: {end_time-start_time}")
rmse_v = nmse(recon_subj, gt_subj)
ssim_v = nmse(recon_subj, gt_subj)
psnr_v = nmse(recon_subj, gt_subj)
print('Subject Done: ', 'RMSE: ', str(rmse_sli), ' SSIM: ', str(ssim_sli),
' PSNR: ', str(psnr_sli))
pickle.dump(
recon_subj,
open(log_path + subj + str(k) + mode + str(restore_sense) + str(R),
'wb'))
end_time = time.perf_counter()
print(f"Elapsed time for {len(subj_loader)} slices: {end_time-start_time}")
import cv2
import numpy as np
import utils
import glob
img_fn = glob.glob(
'/home/didi/Repository/computervision/image-handling/traffic02.jpg')
enlarge = 1
crop_dim = (1242, 375)
for i, img_path in enumerate(img_fn):
img = cv2.imread(img_path)
img_resize = utils.resize(enlarge, img)
img_crop = utils.center_crop(img_resize, crop_dim)
cv2.imwrite(
f'/home/didi/Repository/computervision/image-handling/{i:06d}.png',
img_crop)
print(f"Kitti Size: (375, 1242)")
print(f"Original: {img.shape}")
print(f"Resize: {img_resize.shape}")
print(f"Crop: {img_crop.shape}")
cv2.imshow('resize', img_resize)
cv2.imshow('after_crop', img_crop)
cv2.imshow('before crop', img)
cv2.waitKey(0)
def refresh_images(self):
curr_refims = []
curr_refimids = []
# add images until nrefims satisfied
while len(curr_refims) < self._nrefims:
n_to_show = self._nrefims - len(curr_refims)
# dynamic reference image dir takes precedence
if self._dynrefimdir and os.path.isdir(self._dynrefimdir):
dynref_rps_added = []
dynref_rps_unseen = set(os.listdir(
self._dynrefimdir)) - self._dynref_rps_seen
invalid_rps = set(rp for rp in dynref_rps_unseen
if os.path.splitext(rp)[1].lower() not in
self.image_extensions)
self._dynref_rps_seen |= invalid_rps
dynref_rps_unseen -= invalid_rps
n_dynref_to_show = n_to_show if self._max_n_dynref is None \
else min(n_to_show, self._max_n_dynref - len(curr_refims))
if dynref_rps_unseen and n_dynref_to_show > 0:
for imfn, i in zip(dynref_rps_unseen,
range(n_dynref_to_show)):
self._dynref_rps_seen.add(imfn)
imid = os.path.splitext(imfn)[0]
try: # if already loaded and cached
curr_refims.append(
self._loaded_refims['dyn'][imid])
curr_refimids.append(imid)
dynref_rps_added.append(imfn)
except KeyError:
im = utils.read_image(
os.path.join(self._dynrefimdir, imfn))
if im is not None:
if self._crop_center:
im = utils.center_crop(im)
self._check_add_to_buffer(im,
imid,
is_dyn=True)
curr_refims.append(im)
curr_refimids.append(imid)
dynref_rps_added.append(imfn)
if len(dynref_rps_added) > 0:
print('added dynref images:', dynref_rps_added)
continue # check dyn ref again; only check stat ref if no dyn imids processed
# static reference image dir
for i in range(n_to_show):
idx_toshow = self._all_view[self._i_toshow %
self._n_all_refims]
if self._all_refims_is_valid[idx_toshow]:
rp = self._all_refimrpaths[idx_toshow]
imid = self._all_refimids[idx_toshow]
try:
curr_refims.append(self._loaded_refims['stat'][imid])
curr_refimids.append(imid)
except KeyError: # not found in loaded images
im = utils.read_image(os.path.join(self._refimdir, rp))
if im is None: # not a valid image
self._all_refims_is_valid[idx_toshow] = False
print(
f'{self.__class__.__name__}{self._idx_str}: invalide reference image: {rp}'
)
else:
if self._crop_center:
im = utils.center_crop(im)
self._check_add_to_buffer(im, imid)
curr_refims.append(im)
curr_refimids.append(imid)
# check change of epoch
self._i_toshow += 1
epoch = self._i_toshow / self._n_all_refims
dynref_epoch = epoch * self._dynref_ref_ratio
if epoch >= self._epoch + 1:
self._epoch = int(epoch)
if self._shuffle:
self._random_generator.shuffle(self._all_view)
if dynref_epoch >= self._dynref_epoch + 1:
self._dynref_epoch = int(dynref_epoch)
self._dynref_rps_seen = set()
self._curr_refims = curr_refims
self._curr_refimids = curr_refimids
if self._verbose:
print(
f'{self.__class__.__name__}{self._idx_str}: showing the following {self._nrefims} reference iamges'
)
print(curr_refimids)
def main():
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Create needed dirs
in_dir = os.path.join(args.tmp_dir, 'input')
out_dir = os.path.join(args.tmp_dir, 'sytlized')
if not os.path.exists(in_dir):
os.makedirs(in_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
if os.path.isdir(args.in_path):
in_path = get_files(args.in_path)
else: # Single image file
in_path = [args.in_path]
if os.path.isdir(args.style_path):
style_files = get_files(args.style_path)
else: # Single image file
style_files = [args.style_path]
print(style_files)
# time.sleep(999)
in_args = [
'ffmpeg',
'-i', args.in_path,
'%s/frame_%%d.png' % in_dir
]
subprocess.call(" ".join(in_args), shell=True)
base_names = os.listdir(in_dir)
in_files = [os.path.join(in_dir, x) for x in base_names]
out_files = [os.path.join(out_dir, x) for x in base_names]
s = time.time()
for content_fullpath in in_path:
content_prefix, content_ext = os.path.splitext(content_fullpath)
content_prefix = os.path.basename(content_prefix)
try:
for style_fullpath in style_files:
style_img = get_img(style_fullpath)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
style_prefix, _ = os.path.splitext(style_fullpath)
style_prefix = os.path.basename(style_prefix)
# print("ARRAY: ", style_img)
out_v = os.path.join(args.out_path, '{}_{}{}'.format(content_prefix, style_prefix, content_ext))
print("OUT:", out_v)
if os.path.isfile(out_v):
print("SKIP", out_v)
continue
for in_f, out_f in zip(in_files, out_files):
print('{} -> {}'.format(in_f, out_f))
content_img = get_img(in_f)
if args.keep_colors:
style_rgb = preserve_colors_np(style_img, content_img)
else:
style_rgb = style_img
stylized = wct_model.predict(content_img, style_rgb, args.alpha, args.swap5, args.ss_alpha)
if args.passes > 1:
for _ in range(args.passes-1):
stylized = wct_model.predict(stylized, style_rgb, args.alpha)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(style_rgb, (stylized.shape[0], stylized.shape[0]))
stylized = np.hstack([style_img_resized, stylized])
save_img(out_f, stylized)
fr = 30
out_args = [
'ffmpeg',
'-i', '%s/frame_%%d.png' % out_dir,
'-f', 'mp4',
'-q:v', '0',
'-vcodec', 'mpeg4',
'-r', str(fr),
'"' + out_v + '"'
]
print(out_args)
subprocess.call(" ".join(out_args), shell=True)
print('Video at: %s' % out_v)
if args.keep_tmp is True or len(style_files) > 1:
continue
else:
shutil.rmtree(args.tmp_dir)
print('Processed in:', (time.time() - s))
print('Processed in:', (time.time() - s))
except Exception as e:
print("EXCEPTION: ", e)
def process_single_data(self, text):
"""
Process a single line of text in the data file
INPUT:
text: string a line of text in the data file
img_size: int the output image size
using_rotation bool if to use random rotation to make data enhancement
kernel_size int the size of kernel in gauss kernel
label_size: int the output label size
OUTPUT:
Normalized Image, Normalized Label Image, mask, Box Size, COM
Normalized uvd groundtruth, Heatmap, Normalized Dmap
"""
if self.process_mode == 'uvd':
# decode the text and load the image with only hand and the uvd coordinate of joints
_image, _joint_uvd, _com, cube_size = self.load_from_text(text)
else: # process_mode == 'bb'
assert self.test_only
_image = self.load_from_text_bb(text)
_com = None
cube_size = None
if _com is None:
mean = np.mean(_image[_image > 0])
_com = center_of_mass(_image > 0)
_com = np.array([_com[1], _com[0], mean])
if cube_size is None:
cube_size = self.cube_size
try:
if not self.augmentation:
raise Exception('load data without augmentation')
image = _image.copy()
joint_uvd = _joint_uvd.copy()
com = _com.copy()
if self.using_rotation:
angle = random.random() * 60 - 30 # random rotation [-30, 30]
else:
angle = 0
if self.using_scale:
scale = 0.8 + random.random() * 0.4 # random scale [0.8, 1.2]
else:
scale = 1.0
if self.using_shift:
# random shift [-5, 5]
shift_x = -5 + random.random() * 10
shift_y = -5 + random.random() * 10
com = self.uvd2xyz(com)
com[0] += shift_x
com[1] += shift_y
com = self.xyz2uvd(com)
# crop the image
du = cube_size / com[2] * self.fx
dv = cube_size / com[2] * self.fy
box_size = int(du + dv)
box_size = max(box_size, 2)
crop_img = center_crop(image, (com[1], com[0]), box_size)
crop_img = crop_img * np.logical_and(crop_img > com[2] - cube_size, crop_img < com[2] + cube_size)
# norm the image and uvd to COM
crop_img[crop_img > 0] -= com[2] # center the depth image to COM
com[0] = int(com[0])
com[1] = int(com[1])
box_size = crop_img.shape[0] # update box_size
if self.using_flip:
if random.random() < 0.5: # probality to flip
for j in range(crop_img.shape[1] // 2):
tem = crop_img[:, j].copy()
crop_img[:, j] = crop_img[:, crop_img.shape[1] - j - 1].copy()
crop_img[:, crop_img.shape[1] - j - 1] = tem
joint_uvd_centered[:, 0] = - joint_uvd_centered[:, 0]
# resize the image and uvd
try:
img_resize = cv2.resize(crop_img, (self.image_size, self.image_size))
except:
# probably because size is zero
print("resize error")
raise ValueError("Resize error")
joint_uvd_centered = joint_uvd - com # center the uvd to COM
joint_uvd_centered_resize = joint_uvd_centered.copy()
joint_uvd_centered_resize[:,:2] = joint_uvd_centered_resize[:,:2] / (box_size - 1) * (self.image_size - 1)
# random rotate the image and the label
img_resize, joint_uvd_centered_resize = random_rotated(img_resize, joint_uvd_centered_resize, angle, scale)
# change hand size
img_resize = img_resize * scale
joint_uvd_centered_resize[:, 2] *= scale
# Generate Heatmap
joint_uvd_kernel = joint_uvd_centered_resize.copy()
joint_uvd_kernel[:,:2] = joint_uvd_kernel[:,:2] / (self.image_size - 1) * (self.label_size - 1) + \
np.array([self.label_size // 2, self.label_size // 2])
# try generate heatmaps with augmented data, which may fail
heatmaps = [generate_kernel(generate_heatmap(self.label_size, joint_uvd_kernel[i, 0], joint_uvd_kernel[i, 1]), \
kernel_size=self.kernel_size, sigmoid=self.sigmoid)[:, :, np.newaxis] for i in range(self.joint_number)]
# Generate label_image and mask
label_image = cv2.resize(img_resize, (self.label_size, self.label_size))
is_hand = label_image != 0
mask = is_hand.astype(float)
except: # not performing any data augmentation
image = _image.copy()
com = _com.copy()
# crop the image
du = cube_size / com[2] * self.fx
dv = cube_size / com[2] * self.fy
box_size = int(du + dv)
box_size = max(box_size, 2)
crop_img = center_crop(image, (com[1], com[0]), box_size)
crop_img = crop_img * np.logical_and(crop_img > com[2] - cube_size, crop_img < com[2] + cube_size)
# norm the image and uvd to COM
crop_img[crop_img > 0] -= com[2] # center the depth image to COM
com[0] = int(com[0])
com[1] = int(com[1])
box_size = crop_img.shape[0] # update box_size
# resize the image and uvd
try:
img_resize = cv2.resize(crop_img, (self.image_size, self.image_size))
except:
# probably because size is zero
print("resize error")
raise ValueError("Resize error")
# Generate label_image and mask
label_image = cv2.resize(img_resize, (self.label_size, self.label_size))
is_hand = label_image != 0
mask = is_hand.astype(float)
if self.test_only:
# Just return the basic elements we need to run the network
# normalize the image first before return
normalized_img = img_resize / cube_size
normalized_label_img = label_image / cube_size
# Convert to torch format
normalized_img = torch.from_numpy(normalized_img).float().unsqueeze(0)
normalized_label_img = torch.from_numpy(normalized_label_img).float().unsqueeze(0)
mask = torch.from_numpy(mask).float().unsqueeze(0)
box_size = torch.tensor(box_size).float()
cube_size = torch.tensor(cube_size).float()
com = torch.from_numpy(com).float()
return normalized_img, normalized_label_img, mask, box_size, cube_size, com
joint_uvd = _joint_uvd.copy()
joint_uvd_centered = joint_uvd - com # center the uvd to COM
joint_uvd_centered_resize = joint_uvd_centered.copy()
joint_uvd_centered_resize[:,:2] = joint_uvd_centered_resize[:,:2] / (box_size - 1) * (self.image_size - 1)
# Generate Heatmap
joint_uvd_kernel = joint_uvd_centered_resize.copy()
joint_uvd_kernel[:,:2] = joint_uvd_kernel[:,:2] / (self.image_size - 1) * (self.label_size - 1) + \
np.array([self.label_size // 2, self.label_size // 2])
try:
heatmaps = [generate_kernel(generate_heatmap(self.label_size, joint_uvd_kernel[i, 0], joint_uvd_kernel[i, 1]), \
kernel_size=self.kernel_size, sigmoid=self.sigmoid)[:, :, np.newaxis] for i in range(self.joint_number)]
except:
path, _ = self.decode_line_txt(text)
print("{} heatmap error".format(path))
raise ValueError("{} heatmap error".format(path))
heatmaps = np.concatenate(heatmaps, axis=2)
# Generate Dmap
Dmap = []
for i in range(self.joint_number):
heatmask = heatmaps[:, :, i] > 0
heatmask = heatmask.astype(float) * mask
Dmap.append(((joint_uvd_centered_resize[i, 2] - label_image.copy()) * heatmask)[:, :, np.newaxis])
Dmap = np.concatenate(Dmap, axis=2)
# Normalize data
normalized_img = img_resize / cube_size
normalized_label_img = label_image / cube_size
normalized_Dmap = Dmap / cube_size
normalized_uvd = joint_uvd_centered_resize.copy()
normalized_uvd[:, :2] = normalized_uvd[:, :2] / (self.image_size - 1)
normalized_uvd[:, 2] = normalized_uvd[:, 2] / cube_size
if np.any(np.isnan(normalized_img)) or np.any(np.isnan(normalized_uvd)) or \
np.any(np.isnan(heatmaps)) or np.any(np.isnan(normalized_label_img)) or \
np.any(np.isnan(normalized_Dmap)) or np.any(np.isnan(mask)) or np.sum(mask) < 10:
path, data = self.decode_line_txt(text)
print("Wired things happen, image contain Nan {}, {}".format(path, np.sum(mask)))
raise ValueError("Wired things happen, image contain Nan {}, {}".format(path, np.sum(mask)))
# Convert to torch format
normalized_img = torch.from_numpy(normalized_img).float().unsqueeze(0)
normalized_label_img = torch.from_numpy(normalized_label_img).float().unsqueeze(0)
mask = torch.from_numpy(mask).float().unsqueeze(0)
box_size = torch.tensor(box_size).float()
cube_size = torch.tensor(cube_size).float()
com = torch.from_numpy(com).float()
normalized_uvd = torch.from_numpy(normalized_uvd).float()
heatmaps = torch.from_numpy(heatmaps).float().permute(2, 0, 1).contiguous()
normalized_Dmap = torch.from_numpy(normalized_Dmap).float().permute(2, 0, 1).contiguous()
return normalized_img, normalized_label_img, mask, box_size, cube_size, com, normalized_uvd, heatmaps, normalized_Dmap
def main(_):
pp.pprint(flags.FLAGS.__flags)
if not os.path.exists(FLAGS.checkpoint_dir):
os.makedirs(FLAGS.checkpoint_dir)
if not os.path.exists(FLAGS.sample_dir):
os.makedirs(FLAGS.sample_dir)
if not os.path.exists(os.path.join('./logs', time.strftime('%d%m'))):
os.makedirs(os.path.join('./logs', time.strftime('%d%m')))
gpu_config = tf.GPUOptions(per_process_gpu_memory_fraction=FLAGS.gpu)
#with tf.Session() as sess:
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_config)) as sess:
if FLAGS.is_train:
fcn = FCN(sess, image_size=FLAGS.image_size, batch_size=FLAGS.batch_size,\
dataset_name=FLAGS.dataset,is_crop=FLAGS.is_crop, checkpoint_dir=FLAGS.checkpoint_dir)
else:
fcn = EVAL(sess, batch_size=1,rgb_image_shape=[224,224,3],dataset_name=FLAGS.dataset,\
is_crop=False, checkpoint_dir=FLAGS.checkpoint_dir)
if FLAGS.is_train:
fcn.train(FLAGS)
else:
VGG_mean = [103.939, 116.779, 123.68]
train_rgb_data = open("db/Oxford_data1_RGB_train.txt")
train_rgblist = train_rgb_data.readlines()
train_depth_data = open("db/Oxford_data1_depth_train.txt")
train_depthlist = train_depth_data.readlines()
shuf = range(0, len(train_rgblist))
random.shuffle(shuf)
shuf = shuf[:10]
save_files = glob.glob(
os.path.join(FLAGS.checkpoint_dir, FLAGS.dataset,
'FCN.model*'))
save_files = natsorted(save_files)
savepath = './Depth_seg'
if not os.path.exists(os.path.join(savepath)):
os.makedirs(os.path.join(savepath))
model = save_files[-2]
model = model.split('/')
model = model[-1]
fcn.load(FLAGS.checkpoint_dir, model)
for m in range(len(shuf)):
rgbpath = train_rgblist[shuf[m]]
rgb_img = scipy.misc.imread(rgbpath[:-1]).astype(np.float32)
rgb_img = center_crop(rgb_img, 224)
rgb_img = np.reshape(rgb_img, (1, 224, 224, 3))
depthpath = train_depthlist[shuf[m]]
depth_img = sio.loadmat(depthpath[:-1])
depth_img = depth_img['depth']
depth_img = np.reshape(
depth_img, [depth_img.shape[0], depth_img.shape[1], 1])
depth_img = center_crop(depth_img, 224)
start_time = time.time()
predict = sess.run(fcn.pred_seg,
feed_dict={
fcn.rgb_images: rgb_img,
fcn.keep_prob: 1.0
})
predict = np.squeeze(predict).astype(np.float32)
print('time: %.8f' % (time.time() - start_time))
if not os.path.exists(os.path.join(savepath, '%s' % (model))):
os.makedirs(os.path.join(savepath, '%s' % (model)))
savename = os.path.join(
savepath, '%s/predict_%03d.jpg' % (model, shuf[m]))
scipy.misc.imsave(savename, predict.astype(np.uint8))
savename = os.path.join(savepath,
'%s/gt_%03d.jpg' % (model, shuf[m]))
scipy.misc.imsave(savename,
np.squeeze(depth_img).astype(np.uint8))
def main():
# Load the WCT model
wct_model = WCT(checkpoints=args.checkpoints,
relu_targets=args.relu_targets,
vgg_path=args.vgg_path,
device=args.device,
ss_patch_size=args.ss_patch_size,
ss_stride=args.ss_stride)
# Create needed dirs
in_dir = os.path.join(args.tmp_dir, 'input')
out_dir = os.path.join(args.tmp_dir, 'sytlized')
if not os.path.exists(in_dir):
os.makedirs(in_dir)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
in_args = ['ffmpeg', '-i', args.in_path, '%s/frame_%%d.png' % in_dir]
subprocess.call(" ".join(in_args), shell=True)
base_names = os.listdir(in_dir)
in_files = [os.path.join(in_dir, x) for x in base_names]
out_files = [os.path.join(out_dir, x) for x in base_names]
style_img = get_img(args.style_path)
if args.style_size > 0:
style_img = resize_to(style_img, args.style_size)
if args.crop_size > 0:
style_img = center_crop(style_img, args.crop_size)
s = time.time()
for in_f, out_f in zip(in_files, out_files):
print('{} -> {}'.format(in_f, out_f))
content_img = get_img(in_f)
if args.keep_colors:
style_rgb = preserve_colors_np(style_img, content_img)
else:
style_rgb = style_img
stylized = wct_model.predict(content_img, style_rgb, args.alpha,
args.swap5, args.ss_alpha)
if args.passes > 1:
for _ in range(args.passes - 1):
stylized = wct_model.predict(stylized, style_rgb, args.alpha)
# Stitch the style + stylized output together, but only if there's one style image
if args.concat:
# Resize style img to same height as frame
style_img_resized = scipy.misc.imresize(
style_rgb, (stylized.shape[0], stylized.shape[0]))
stylized = np.hstack([style_img_resized, stylized])
save_img(out_f, stylized)
fr = 30
out_args = [
'ffmpeg', '-i',
'%s/frame_%%d.png' % out_dir, '-f', 'mp4', '-q:v', '0', '-vcodec',
'mpeg4', '-r',
str(fr), args.out_path
]
subprocess.call(" ".join(out_args), shell=True)
print('Video at: %s' % args.out_path)
if args.keep_tmp is False:
shutil.rmtree(args.tmp_dir)
print('Processed in:', (time.time() - s))
query_fn = cfg['qimlist'][i]
print(str(i)+" QUERY "+query_fn); log.write(str(i)+" QUERY "+query_fn+"\n"); log.flush()
# load target (query) image
if bbxs is not None:
target_img = img_loader(cfg['qim_fname'](cfg,i), im_size[dataset], cfg['gnd'][i]['bbx']).type(torch.cuda.FloatTensor)
else:
target_img = img_loader(cfg['qim_fname'](cfg,i), im_size[dataset]).type(torch.cuda.FloatTensor)
# attack
t = time.time()
trials = 0
converged = False
while not converged and trials < max_trials:
carrier_img = img_loader("data/input/"+carrier_fn+".jpg", im_size[dataset]).type(torch.cuda.FloatTensor)
carrier_img = center_crop(target_img, carrier_img)
alr = lr / divide_rate_lr**trials # reducing lr after every failure
aiters = int(iters * multiply_rate_iters**trials) # increase iterations after every failure
attack_img, loss_perf, loss_distort, converged = tma(train_networks, scale_factors, target_img, carrier_img, mode = mode, num_steps = aiters, lr = alr, lam = lam, sigma_blur = sigma_blur, verbose = True)
trials += 1
# time and log
total_time += time.time()-t
log.write("performance loss {:6f} distortion loss {:6f} total loss {:6f}\n".format(loss_perf.item(), (loss_distort).item(), (loss_distort+loss_perf).item())); log.flush()
if trials == max_trials: print("Failed...")
# save the attack in png file
savefn = output_folder+exp_name+"/"+query_fn+'.png'
if not os.path.exists(savefn[0:savefn.rfind('/')]): os.makedirs(savefn[0:savefn.rfind('/')])
imsave(savefn,np.transpose(attack_img.cpu().numpy(), (2,3,1,0)).squeeze())
print("Attack saved in "+savefn+"\n"); sys.stdout.flush()
