def run_pix2pix(img, net, opt):
if opt.netG == 'HD':
img = impro.resize(img, 512)
else:
img = impro.resize(img, 128)
img = data.im2tensor(img, use_gpu=opt.use_gpu)
img_fake = net(img)
img_fake = data.tensor2im(img_fake)
return img_fake
def cleanmosaic_video_fusion(opt,netG,netM):
path = opt.media_path
N = 25
INPUT_SIZE = 128
fps,imagepaths = video_init(opt,path)
positions = []
# get position
for i,imagepath in enumerate(imagepaths,1):
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
# x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3]
x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
cv2.imwrite(os.path.join('./tmp/mosaic_mask',imagepath), mask)
positions.append([x,y,size])
print('\r','Find mosaic location:'+str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print('\nOptimize mosaic locations...')
positions =np.array(positions)
for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num)
# clean mosaic
for i,imagepath in enumerate(imagepaths,0):
x,y,size = positions[i][0],positions[i][1],positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
mask = cv2.imread(os.path.join('./tmp/mosaic_mask',imagepath),0)
if size==0:
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_origin)
else:
mosaic_input = np.zeros((INPUT_SIZE,INPUT_SIZE,3*N+1), dtype='uint8')
for j in range(0,N):
img = impro.imread(os.path.join('./tmp/video2image',imagepaths[np.clip(i+j-12,0,len(imagepaths)-1)]))
img = img[y-size:y+size,x-size:x+size]
img = impro.resize(img,INPUT_SIZE)
mosaic_input[:,:,j*3:(j+1)*3] = img
mask = impro.resize(mask,np.min(img_origin.shape[:2]))
mask = mask[y-size:y+size,x-size:x+size]
mask = impro.resize(mask, INPUT_SIZE)
mosaic_input[:,:,-1] = mask
mosaic_input = data.im2tensor(mosaic_input,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False)
unmosaic_pred = netG(mosaic_input)
#unmosaic_pred = (unmosaic_pred.cpu().detach().numpy()*255)[0]
#img_fake = unmosaic_pred.transpose((1, 2, 0))
img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = False ,is0_1 = False)
img_result = impro.replace_mosaic(img_origin,img_fake,x,y,size,opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_result)
print('\r','Clean Mosaic:'+str(i+1)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print()
ffmpeg.image2video( fps,
'./tmp/replace_mosaic/output_%05d.'+opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4'))
def loadimage(imagepaths,maskpaths,opt,test_flag = False):
batchsize = len(imagepaths)
images = np.zeros((batchsize,3,opt.finesize,opt.finesize), dtype=np.float32)
masks = np.zeros((batchsize,1,opt.finesize,opt.finesize), dtype=np.float32)
for i in range(len(imagepaths)):
img = impro.resize(impro.imread(imagepaths[i]),opt.loadsize)
mask = impro.resize(impro.imread(maskpaths[i],mod = 'gray'),opt.loadsize)
img,mask = data.random_transform_image(img, mask, opt.finesize, test_flag)
images[i] = (img.transpose((2, 0, 1))/255.0)
masks[i] = (mask.reshape(1,1,opt.finesize,opt.finesize)/255.0)
images = Totensor(images,opt.use_gpu)
masks = Totensor(masks,opt.use_gpu)
return images,masks
def run_unet_rectim(img, net, size=128, use_gpu=True):
img = impro.resize(img, size)
img1, img2 = impro.spiltimage(img)
mask1 = run_unet(img1, net, size=128, use_gpu=use_gpu)
mask2 = run_unet(img2, net, size=128, use_gpu=use_gpu)
mask = impro.mergeimage(mask1, mask2, img)
return mask
def run_styletransfer(opt, net, img):
if opt.output_size != 0:
if 'resize' in opt.preprocess and 'resize_scale_width' not in opt.preprocess:
img = impro.resize(img, opt.output_size)
elif 'resize_scale_width' in opt.preprocess:
img = cv2.resize(img, (opt.output_size, opt.output_size))
img = img[0:4 * int(img.shape[0] / 4), 0:4 * int(img.shape[1] / 4), :]
if 'edges' in opt.preprocess:
if opt.canny > 100:
canny_low = opt.canny - 50
canny_high = np.clip(opt.canny + 50, 0, 255)
elif opt.canny < 50:
canny_low = np.clip(opt.canny - 25, 0, 255)
canny_high = opt.canny + 25
else:
canny_low = opt.canny - int(opt.canny / 2)
canny_high = opt.canny + int(opt.canny / 2)
img = cv2.Canny(img, opt.canny - 50, opt.canny + 50)
if opt.only_edges:
return img
img = data.im2tensor(img,
use_gpu=opt.use_gpu,
gray=True,
use_transform=False,
is0_1=False)
else:
img = data.im2tensor(img, use_gpu=opt.use_gpu)
img = net(img)
img = data.tensor2im(img)
return img
def handle():
result = {}
# to opencv img
try:
imgRec = request.form['img']
imgByte = base64.b64decode(imgRec)
img_np_arr = np.frombuffer(imgByte, np.uint8)
img = cv2.imdecode(img_np_arr, cv2.IMREAD_COLOR)
except Exception as e:
result['img'] = imgRec
result['info'] = 'readfailed'
return result
# run model
try:
if max(img.shape) > 1080:
img = impro.resize(img, 720, interpolation=cv2.INTER_CUBIC)
img = core.cleanmosaic_img_server(opt, img, netG, netM)
except Exception as e:
result['img'] = imgRec
result['info'] = 'procfailed'
return result
# return
imgbytes = cv2.imencode('.jpg', img)[1]
imgString = base64.b64encode(imgbytes).decode('utf-8')
result['img'] = imgString
result['info'] = 'ok'
return result
def cleanmosaic_video_fusion(opt,netG,netM):
path = opt.media_path
N = 25
if 'HD' in os.path.basename(opt.model_path):
INPUT_SIZE = 256
else:
INPUT_SIZE = 128
fps,imagepaths,height,width = video_init(opt,path)
positions = get_mosaic_positions(opt,netM,imagepaths,savemask=True)
# clean mosaic
img_pool = np.zeros((height,width,3*N), dtype='uint8')
for i,imagepath in enumerate(imagepaths,0):
x,y,size = positions[i][0],positions[i][1],positions[i][2]
# image read stream
mask = cv2.imread(os.path.join('./tmp/mosaic_mask',imagepath),0)
if i==0 :
for j in range(0,N):
img_pool[:,:,j*3:(j+1)*3] = impro.imread(os.path.join('./tmp/video2image',imagepaths[np.clip(i+j-12,0,len(imagepaths)-1)]))
else:
img_pool[:,:,0:(N-1)*3] = img_pool[:,:,3:N*3]
img_pool[:,:,(N-1)*3:] = impro.imread(os.path.join('./tmp/video2image',imagepaths[np.clip(i+12,0,len(imagepaths)-1)]))
img_origin = img_pool[:,:,int((N-1)/2)*3:(int((N-1)/2)+1)*3]
if size==0: # can not find mosaic,
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_origin)
else:
mosaic_input = np.zeros((INPUT_SIZE,INPUT_SIZE,3*N+1), dtype='uint8')
mosaic_input[:,:,0:N*3] = impro.resize(img_pool[y-size:y+size,x-size:x+size,:], INPUT_SIZE)
mask_input = impro.resize(mask,np.min(img_origin.shape[:2]))[y-size:y+size,x-size:x+size]
mosaic_input[:,:,-1] = impro.resize(mask_input, INPUT_SIZE)
mosaic_input = data.im2tensor(mosaic_input,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False)
unmosaic_pred = netG(mosaic_input)
img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = False ,is0_1 = False)
img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_result)
print('\r','Clean Mosaic:'+str(i+1)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=35),end='')
print()
ffmpeg.image2video( fps,
'./tmp/replace_mosaic/output_%05d.'+opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4'))
def run_segment(img, net, size=360, use_gpu=0):
img = impro.resize(img, size)
img = data.im2tensor(img,
use_gpu=use_gpu,
bgr2rgb=False,
use_transform=False,
is0_1=True)
mask = net(img)
mask = data.tensor2im(mask, gray=True, rgb2bgr=False, is0_1=True)
return mask
def gen_base64():
if 'image' not in request.json:
return error('Stupid request'), 412
if 'base64' in request.args:
use_base64 = True if request.args.get('base64') == 'true' else False
else:
use_base64 = False
image_data = str.encode(request.json['image'])
image_data = image_data[23:]
ip = request.environ.get('HTTP_X_REAL_IP', request.remote_addr)
timestamp = datetime.datetime.now().isoformat()
image_name = ip + '_' + timestamp + '.png'
image_path = os.path.join(app.config['UPLOAD_FOLDER'], image_name)
with open(image_path, "wb") as f:
f.write(base64.decodebytes(image_data))
image = cv2.imread(image_path)
image = resize(image, 256, 256)
cv2.imwrite(image_path, image)
## Load image and begin generating
real = Image.open(image_path)
preprocess = transforms.Compose([
transforms.Scale(opt.loadSize),
transforms.RandomCrop(opt.fineSize),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
# Load input
input_A = preprocess(real).unsqueeze_(0)
model.input_A.resize_(input_A.size()).copy_(input_A)
# Forward (model.real_A) through G and produce output (model.fake_B)
model.test()
# Convert image to numpy array
fake = util.tensor2im(model.fake_B.data)
output_path = os.path.join(app.config['GAN_FOLDER'], image_name)
# Save image
util.save_image(fake, output_path)
# if not use_base64:
# return send_file(output_path)
# image = open(output_path, 'rb').read()
# encoded = 'data:image/jpeg;base64,' + base64.b64encode(image).decode('utf-8')
# return encoded
return '/cgan/' + image_name
def loaddata():
video_index = random.randint(0,len(videos)-1)
video = videos[video_index]
img_index = random.randint(int(N/2)+1,lengths[video_index]- int(N/2)-1)
input_img = np.zeros((opt.loadsize,opt.loadsize,3*N+1), dtype='uint8')
for i in range(0,N):
img = cv2.imread('./dataset/'+video+'/mosaic/output_'+'%05d'%(img_index+i-int(N/2))+'.png')
img = impro.resize(img,opt.loadsize)
input_img[:,:,i*3:(i+1)*3] = img
mask = cv2.imread('./dataset/'+video+'/mask/output_'+'%05d'%(img_index)+'.png',0)
mask = impro.resize(mask,opt.loadsize)
mask = impro.mask_threshold(mask,15,128)
input_img[:,:,-1] = mask
ground_true = cv2.imread('./dataset/'+video+'/ori/output_'+'%05d'%(img_index)+'.png')
ground_true = impro.resize(ground_true,opt.loadsize)
input_img,ground_true = data.random_transform_video(input_img,ground_true,opt.finesize,N)
input_img = data.im2tensor(input_img,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1=False)
ground_true = data.im2tensor(ground_true,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1=False)
return input_img,ground_true
def replace_mosaic(img_origin,img_fake,x,y,size,no_father = opt.no_feather):
img_fake = impro.resize(img_fake,size*2)
if no_father:
img_origin[y-size:y+size,x-size:x+size]=img_fake
img_result = img_origin
else:
eclosion_num = int(size/5)
entad = int(eclosion_num/2+2)
mask = np.zeros(img_origin.shape, dtype='uint8')
mask = cv2.rectangle(mask,(x-size+entad,y-size+entad),(x+size-entad,y+size-entad),(255,255,255),-1)
mask = (cv2.blur(mask, (eclosion_num, eclosion_num)))
mask = mask/255.0
img_tmp = np.zeros(img_origin.shape)
img_tmp[y-size:y+size,x-size:x+size]=img_fake
img_result = img_origin.copy()
img_result = (img_origin*(1-mask)+img_tmp*mask).astype('uint8')
return img_result
def cleanmosaic_video_fusion(opt, netG, netM):
path = opt.media_path
N = 25
if 'HD' in os.path.basename(opt.model_path):
INPUT_SIZE = 256
else:
INPUT_SIZE = 128
fps, imagepaths, height, width = video_init(opt, path)
positions = get_mosaic_positions(opt, netM, imagepaths, savemask=True)
t1 = time.time()
if not opt.no_preview:
cv2.namedWindow('clean', cv2.WINDOW_NORMAL)
# clean mosaic
print('Clean Mosaic:')
length = len(imagepaths)
img_pool = np.zeros((height, width, 3 * N), dtype='uint8')
mosaic_input = np.zeros((INPUT_SIZE, INPUT_SIZE, 3 * N + 1), dtype='uint8')
for i, imagepath in enumerate(imagepaths, 0):
x, y, size = positions[i][0], positions[i][1], positions[i][2]
# image read stream
mask = cv2.imread(
os.path.join(opt.temp_dir + '/mosaic_mask', imagepath), 0)
if i == 0:
for j in range(0, N):
img_pool[:, :, j * 3:(j + 1) * 3] = impro.imread(
os.path.join(
opt.temp_dir + '/video2image',
imagepaths[np.clip(i + j - 12, 0,
len(imagepaths) - 1)]))
else:
img_pool[:, :, 0:(N - 1) * 3] = img_pool[:, :, 3:N * 3]
img_pool[:, :, (N - 1) * 3:] = impro.imread(
os.path.join(
opt.temp_dir + '/video2image',
imagepaths[np.clip(i + 12, 0,
len(imagepaths) - 1)]))
img_origin = img_pool[:, :,
int((N - 1) / 2) * 3:(int((N - 1) / 2) + 1) * 3]
img_result = img_origin.copy()
if size > 100:
try: #Avoid unknown errors
#reshape to network input shape
mosaic_input[:, :, 0:N * 3] = impro.resize(
img_pool[y - size:y + size, x - size:x + size, :],
INPUT_SIZE)
mask_input = impro.resize(mask, np.min(
img_origin.shape[:2]))[y - size:y + size,
x - size:x + size]
mosaic_input[:, :, -1] = impro.resize(mask_input, INPUT_SIZE)
mosaic_input_tensor = data.im2tensor(mosaic_input,
bgr2rgb=False,
use_gpu=opt.use_gpu,
use_transform=False,
is0_1=False)
unmosaic_pred = netG(mosaic_input_tensor)
img_fake = data.tensor2im(unmosaic_pred,
rgb2bgr=False,
is0_1=False)
img_result = impro.replace_mosaic(img_origin, img_fake, mask,
x, y, size, opt.no_feather)
except Exception as e:
print('Warning:', e)
cv2.imwrite(os.path.join(opt.temp_dir + '/replace_mosaic', imagepath),
img_result)
#preview result and print
if not opt.no_preview:
cv2.imshow('clean', img_result)
cv2.waitKey(1) & 0xFF
t2 = time.time()
print('\r',
str(i + 1) + '/' + str(length),
util.get_bar(100 * i / length, num=35),
util.counttime(t1, t2, i + 1, len(imagepaths)),
end='')
print()
if not opt.no_preview:
cv2.destroyAllWindows()
ffmpeg.image2video(
fps,
opt.temp_dir + '/replace_mosaic/output_%06d.' + opt.tempimage_type,
opt.temp_dir + '/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
x, y, size, area = impro.boundingSquare(mask, Ex_mul=ex_mul)
positions.append([x, y, size])
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i],
opt.medfilt_num)
for i, imagepath in enumerate(imagepaths):
x, y, size = positions[i][0], positions[i][1], positions[i][2]
tmp_cnt = i
while size < opt.minsize // 2:
tmp_cnt = tmp_cnt - 1
x, y, size = positions[tmp_cnt][0], positions[tmp_cnt][
1], positions[tmp_cnt][2]
img = impro.resize(imgs[i][y - size:y + size,
x - size:x + size],
opt.outsize,
interpolation=cv2.INTER_CUBIC)
mask = impro.resize(masks[i][y - size:y + size,
x - size:x + size],
opt.outsize,
interpolation=cv2.INTER_CUBIC)
impro.imwrite(
os.path.join(origindir, '%05d' % (i + 1) + '.jpg'), img)
impro.imwrite(os.path.join(maskdir, '%05d' % (i + 1) + '.png'),
mask)
# x_tmp,y_tmp,size_tmp
# for imagepath in imagepaths:
# img = impro.imread(imagepath)
# mask,x,y,halfsize,area = runmodel.get_ROI_position(img,net,opt,keepsize=True)
# if halfsize>opt.minsize//4:
#test
if os.path.isdir('./test'):
netG.eval()
test_names = os.listdir('./test')
test_names.sort()
result = np.zeros((opt.finesize*2,opt.finesize*len(test_names),3), dtype='uint8')
for cnt,test_name in enumerate(test_names,0):
img_names = os.listdir(os.path.join('./test',test_name,'image'))
img_names.sort()
inputdata = np.zeros((opt.finesize,opt.finesize,3*N+1), dtype='uint8')
for i in range(0,N):
img = impro.imread(os.path.join('./test',test_name,'image',img_names[i]))
img = impro.resize(img,opt.finesize)
inputdata[:,:,i*3:(i+1)*3] = img
mask = impro.imread(os.path.join('./test',test_name,'mask.png'),'gray')
mask = impro.resize(mask,opt.finesize)
mask = impro.mask_threshold(mask,15,128)
inputdata[:,:,-1] = mask
result[0:opt.finesize,opt.finesize*cnt:opt.finesize*(cnt+1),:] = inputdata[:,:,int((N-1)/2)*3:(int((N-1)/2)+1)*3]
inputdata = data.im2tensor(inputdata,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False)
pred = netG(inputdata)
pred = data.tensor2im(pred,rgb2bgr = False, is0_1 = False)
result[opt.finesize:opt.finesize*2,opt.finesize*cnt:opt.finesize*(cnt+1),:] = pred
cv2.imwrite(os.path.join(dir_checkpoint,str(iter+1)+'_test.jpg'), result)
netG.train()
cnt = 0
for file in filepaths:
try:
cnt += 1
img = impro.imread(file, loadsize=512)
img_drawn = img.copy()
cv2.namedWindow('image')
cv2.setMouseCallback('image', draw_circle) #MouseCallback
while (1):
cv2.imshow('image', img_drawn)
k = cv2.waitKey(1) & 0xFF
if k == ord('s'):
img_drawn = impro.resize(img_drawn, 256)
mask = makemask(img_drawn)
cv2.imwrite(
os.path.join(
mask_savedir,
os.path.splitext(os.path.basename(file))[0] + '.png'),
mask)
cv2.imwrite(os.path.join(img_savedir, os.path.basename(file)),
img)
print(
'Saved:',
os.path.join(
mask_savedir,
os.path.splitext(os.path.basename(file))[0] + '.png'),
mask)
# cv2.destroyAllWindows()
import numpy as np
import cv2
import os
import sys
sys.path.append("..")
from util import image_processing as impro
from util import util
img_dir = './datasets_img/pix2pix/edges2cat/images'
output_dir = './datasets_img/pix2pix/edges2cat/train'
util.makedirs(output_dir)
img_names = os.listdir(img_dir)
for i, img_name in enumerate(img_names, 2000):
try:
img = impro.imread(os.path.join(img_dir, img_name))
img = impro.resize(img, 286)
h, w = img.shape[:2]
edges = cv2.Canny(img, 150, 250)
edges = impro.ch_one2three(edges)
out_img = np.zeros((h, w * 2, 3), dtype=np.uint8)
out_img[:, 0:w] = edges
out_img[:, w:2 * w] = img
cv2.imwrite(os.path.join(output_dir, '%05d' % i + '.jpg'), out_img)
except Exception as e:
pass
def cleanmosaic_video_fusion(opt, netG, netM):
path = opt.media_path
N, T, S = 2, 5, 3
LEFT_FRAME = (N * S)
POOL_NUM = LEFT_FRAME * 2 + 1
INPUT_SIZE = 256
FRAME_POS = np.linspace(0, (T - 1) * S, T, dtype=np.int64)
img_pool = []
previous_frame = None
init_flag = True
fps, imagepaths, height, width = video_init(opt, path)
positions = get_mosaic_positions(opt, netM, imagepaths, savemask=True)
t1 = time.time()
if not opt.no_preview:
cv2.namedWindow('clean', cv2.WINDOW_NORMAL)
# clean mosaic
print('Step:3/4 -- Clean Mosaic:')
length = len(imagepaths)
for i, imagepath in enumerate(imagepaths, 0):
x, y, size = positions[i][0], positions[i][1], positions[i][2]
input_stream = []
# image read stream
if i == 0: # init
for j in range(POOL_NUM):
img_pool.append(
impro.imread(
os.path.join(
opt.temp_dir + '/video2image',
imagepaths[np.clip(i + j - LEFT_FRAME, 0,
len(imagepaths) - 1)])))
else: # load next frame
img_pool.pop(0)
img_pool.append(
impro.imread(
os.path.join(
opt.temp_dir + '/video2image',
imagepaths[np.clip(i + LEFT_FRAME, 0,
len(imagepaths) - 1)])))
img_origin = img_pool[LEFT_FRAME]
img_result = img_origin.copy()
if size > 50:
try: #Avoid unknown errors
for pos in FRAME_POS:
input_stream.append(
impro.resize(
img_pool[pos][y - size:y + size,
x - size:x + size],
INPUT_SIZE)[:, :, ::-1])
if init_flag:
init_flag = False
previous_frame = input_stream[N]
previous_frame = data.im2tensor(previous_frame,
bgr2rgb=True,
gpu_id=opt.gpu_id)
input_stream = np.array(input_stream).reshape(
1, T, INPUT_SIZE, INPUT_SIZE, 3).transpose((0, 4, 1, 2, 3))
input_stream = data.to_tensor(data.normalize(input_stream),
gpu_id=opt.gpu_id)
with torch.no_grad():
unmosaic_pred = netG(input_stream, previous_frame)
img_fake = data.tensor2im(unmosaic_pred, rgb2bgr=True)
previous_frame = unmosaic_pred
# previous_frame = data.tensor2im(unmosaic_pred,rgb2bgr = True)
mask = cv2.imread(
os.path.join(opt.temp_dir + '/mosaic_mask', imagepath), 0)
img_result = impro.replace_mosaic(img_origin, img_fake, mask,
x, y, size, opt.no_feather)
except Exception as e:
init_flag = True
print('Error:', e)
else:
init_flag = True
cv2.imwrite(os.path.join(opt.temp_dir + '/replace_mosaic', imagepath),
img_result)
os.remove(os.path.join(opt.temp_dir + '/video2image', imagepath))
#preview result and print
if not opt.no_preview:
cv2.imshow('clean', img_result)
cv2.waitKey(1) & 0xFF
t2 = time.time()
print('\r',
str(i + 1) + '/' + str(length),
util.get_bar(100 * i / length, num=35),
util.counttime(t1, t2, i + 1, len(imagepaths)),
end='')
print()
if not opt.no_preview:
cv2.destroyAllWindows()
print('Step:4/4 -- Convert images to video')
ffmpeg.image2video(
fps,
opt.temp_dir + '/replace_mosaic/output_%06d.' + opt.tempimage_type,
opt.temp_dir + '/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def gen_photo_nhat():
if 'file' not in request.files:
return error('file form-data not existed'), 412
if 'base64' in request.args:
use_base64 = True if request.args.get('base64') == 'true' else False
else:
use_base64 = False
image = request.files['file']
# Submit taylor.jpg ---> taylor_1234567.jpg (name + timestamp)
image_name, ext = image.filename.rsplit('.', 1)
image_name = image_name + '_' + str(int(time.time())) + '.' + ext
# Save image to /upload
image_path = os.path.join(app.config['UPLOAD_FOLDER'], image_name)
image.save(image_path)
## Crop here
found_face, position = get_face_position(image_path)
if not found_face:
return 'No face found', 404
top = position['top']
bottom = position['bottom']
left = position['left']
right = position['right']
img = cv2.imread(image_path)
img = img[top:bottom, left:right]
img = resize(img, 256, 256)
cv2.imwrite(image_path, img)
## Load image and begin generating
real = Image.open(image_path)
preprocess = transforms.Compose([
transforms.Scale(opt.loadSize),
transforms.RandomCrop(opt.fineSize),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
])
# Load input
input_A = preprocess(real).unsqueeze_(0)
model.input_A.resize_(input_A.size()).copy_(input_A)
# Forward (model.real_A) through G and produce output (model.fake_B)
model.test()
# Convert image to numpy array
fake = util.tensor2im(model.fake_B.data)
output_path = os.path.join(app.config['GAN_FOLDER'], image_name)
# Save image
util.save_image(fake, output_path)
if not use_base64:
return send_file(output_path)
image = open(output_path, 'rb').read()
encoded = 'data:image/jpeg;base64,' + base64.b64encode(image).decode(
'utf-8')
return encoded
def cleanmosaic_video_fusion(opt):
net = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
N = 25
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
# x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3]
x, y, size, mask = runmodel.get_mosaic_position(
img_origin, net_mosaic_pos, opt)
cv2.imwrite(os.path.join('./tmp/mosaic_mask', imagepath), mask)
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
# clean mosaic
print('Clean mosaic...')
for i, imagepath in enumerate(imagepaths, 0):
print('Clean mosaic:', imagepath)
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
mask = cv2.imread(os.path.join('./tmp/mosaic_mask', imagepath), 0)
if size == 0:
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_origin)
else:
mosaic_input = np.zeros((256, 256, 3 * N + 1), dtype='uint8')
for j in range(0, N):
img = impro.imread(
os.path.join(
'./tmp/video2image',
imagepaths[np.clip(i + j - 12, 0,
len(imagepaths) - 1)]))
img = img[y - size:y + size, x - size:x + size]
img = impro.resize(img, 256)
mosaic_input[:, :, j * 3:(j + 1) * 3] = img
mask = impro.resize(mask, np.min(img_origin.shape[:2]))
mask = mask[y - size:y + size, x - size:x + size]
mask = impro.resize(mask, 256)
mosaic_input[:, :, -1] = mask
mosaic_input = data.im2tensor(mosaic_input,
bgr2rgb=False,
use_gpu=opt.use_gpu,
use_transform=False)
unmosaic_pred = net(mosaic_input)
unmosaic_pred = (unmosaic_pred.cpu().detach().numpy() * 255)[0]
img_fake = unmosaic_pred.transpose((1, 2, 0))
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_result)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def run_pix2pix(img, net, size=128, use_gpu=True):
img = impro.resize(img, size)
img = data.im2tensor(img, use_gpu=use_gpu)
img_fake = net(img)
img_fake = data.tensor2im(img_fake)
return img_fake
def run_segment(img, net, size=360, gpu_id='-1'):
img = impro.resize(img, size)
img = data.im2tensor(img, gpu_id=gpu_id, bgr2rgb=False, is0_1=True)
mask = net(img)
mask = data.tensor2im(mask, gray=True, is0_1=True)
return mask
img_names = os.listdir(img_dir)
mask_names.sort()
img_names.sort()
print('Find images:', len(img_names))
cnt = 0
for fold in range(FOLD_NUM):
for img_name, mask_name in zip(img_names, mask_names):
try:
img = impro.imread(os.path.join(img_dir, img_name))
mask = impro.imread(os.path.join(mask_dir, mask_name), 'gray')
mask = impro.resize_like(mask, img)
x, y, size, area = impro.boundingSquare(mask, 1.5)
if area > 100:
if Bounding:
img = impro.resize(
img[y - size:y + size, x - size:x + size], OUT_SIZE)
mask = impro.resize(
mask[y - size:y + size, x - size:x + size], OUT_SIZE)
img_mosaic = mosaic.addmosaic_random(img, mask)
if HD:
cv2.imwrite(
os.path.join(train_A_path, '%05d' % cnt + '.jpg'),
img_mosaic)
cv2.imwrite(
os.path.join(train_B_path, '%05d' % cnt + '.jpg'), img)
else:
merge_img = impro.makedataset(img_mosaic, img)
cv2.imwrite(
os.path.join(train_path, '%05d' % cnt + '.jpg'),
merge_img)
netG.eval()
test_names = os.listdir('./test')
test_names.sort()
result = np.zeros((finesize * 2, finesize * len(test_names), 3),
dtype='uint8')
for cnt, test_name in enumerate(test_names, 0):
img_names = os.listdir(os.path.join('./test', test_name, 'image'))
img_names.sort()
inputdata = np.zeros((finesize, finesize, 3 * N + 1),
dtype='uint8')
for i in range(0, N):
img = impro.imread(
os.path.join('./test', test_name, 'image', img_names[i]))
img = impro.resize(img, finesize)
inputdata[:, :, i * 3:(i + 1) * 3] = img
mask = impro.imread(os.path.join('./test', test_name, 'mask.png'),
'gray')
mask = impro.resize(mask, finesize)
mask = impro.mask_threshold(mask, 15, 128)
inputdata[:, :, -1] = mask
result[0:finesize, finesize * cnt:finesize *
(cnt + 1), :] = inputdata[:, :,
int((N - 1) / 2) *
3:(int((N - 1) / 2) + 1) * 3]
inputdata = data.im2tensor(inputdata,
bgr2rgb=False,
use_gpu=opt.use_gpu,
use_transform=False,
net = loadmodel.unet(opt)
for path in videos:
path = os.path.join('./video', path)
util.clean_tempfiles()
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
positions = []
img_ori_example = impro.imread(
os.path.join('./tmp/video2image', imagepaths[0]))
mask_avg = np.zeros((impro.resize(img_ori_example, 128)).shape[:2])
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
print('Find ROI location:', imagepath)
img = impro.imread(imagepath)
x, y, size, mask = runmodel.get_mosaic_position(img,
net,
opt,
threshold=64)
cv2.imwrite(
os.path.join('./tmp/ROI_mask', os.path.basename(imagepath)), mask)
positions.append([x, y, size])
mask_avg = mask_avg + mask
print('Optimize ROI locations...')
mask_index = filt.position_medfilt(np.array(positions), 13)
for i in range(len(masks)):
masks[i] = masks[i].replace('.png', '.jpg')
for file in files:
if file in masks:
files_new.remove(file)
files = files_new
# files = list(set(files)) #Distinct
print('remain:', len(files))
random.shuffle(files)
# files.sort()
cnt = 0
for file in files:
cnt += 1
img = cv2.imread(os.path.join(image_dir, file))
img = impro.resize(img, 512)
cv2.namedWindow('image')
cv2.setMouseCallback('image', draw_circle) #MouseCallback
while (1):
cv2.imshow('image', img)
k = cv2.waitKey(1) & 0xFF
if k == ord(' '):
img = impro.resize(img, 256)
mask = makemask(img)
cv2.imwrite(
os.path.join(mask_dir,
os.path.splitext(file)[0] + '.png'), mask)
print(os.path.join(mask_dir, os.path.splitext(file)[0] + '.png'))
# cv2.destroyAllWindows()
print('remain:', len(files) - cnt)