def cleanmosaic_video_byframe(opt,netG,netM):
path = opt.media_path
fps,imagepaths = video_init(opt,path)
positions = []
# get position
for i,imagepath in enumerate(imagepaths,1):
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
x,y,size = runmodel.get_mosaic_position(img_origin,netM,opt)[:3]
positions.append([x,y,size])
print('\r','Find mosaic location:'+str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print('\nOptimize mosaic locations...')
positions =np.array(positions)
for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num)
# clean mosaic
for i,imagepath in enumerate(imagepaths,0):
x,y,size = positions[i][0],positions[i][1],positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y-size:y+size,x-size:x+size]
img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
img_result = impro.replace_mosaic(img_origin,img_fake,x,y,size,opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_result)
print('\r','Clean Mosaic:'+str(i+1)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print()
ffmpeg.image2video( fps,
'./tmp/replace_mosaic/output_%05d.'+opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4'))
def get_mosaic_positions(opt, netM, imagepaths, savemask=True):
# get mosaic position
positions = []
t1 = time.time()
if not opt.no_preview:
cv2.namedWindow('mosaic mask', cv2.WINDOW_NORMAL)
print('Step:2/4 -- Find mosaic location')
for i, imagepath in enumerate(imagepaths, 1):
img_origin = impro.imread(
os.path.join(opt.temp_dir + '/video2image', imagepath))
x, y, size, mask = runmodel.get_mosaic_position(img_origin, netM, opt)
positions.append([x, y, size])
if savemask:
cv2.imwrite(os.path.join(opt.temp_dir + '/mosaic_mask', imagepath),
mask)
#preview result and print
if not opt.no_preview:
cv2.imshow('mosaic mask', mask)
cv2.waitKey(1) & 0xFF
t2 = time.time()
print('\r',
str(i) + '/' + str(len(imagepaths)),
util.get_bar(100 * i / len(imagepaths), num=35),
util.counttime(t1, t2, i, len(imagepaths)),
end='')
if not opt.no_preview:
cv2.destroyAllWindows()
print('\nOptimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
return positions
def cleanmosaic_img_server(opt,img_origin,netG,netM):
x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
img_result = img_origin.copy()
if size > 100 :
img_mosaic = img_origin[y-size:y+size,x-size:x+size]
if opt.traditional:
img_fake = runmodel.traditional_cleaner(img_mosaic,opt)
else:
img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
return img_result
def cleanmosaic_video_fusion(opt,netG,netM):
path = opt.media_path
N = 25
INPUT_SIZE = 128
fps,imagepaths = video_init(opt,path)
positions = []
# get position
for i,imagepath in enumerate(imagepaths,1):
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
# x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3]
x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
cv2.imwrite(os.path.join('./tmp/mosaic_mask',imagepath), mask)
positions.append([x,y,size])
print('\r','Find mosaic location:'+str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print('\nOptimize mosaic locations...')
positions =np.array(positions)
for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num)
# clean mosaic
for i,imagepath in enumerate(imagepaths,0):
x,y,size = positions[i][0],positions[i][1],positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
mask = cv2.imread(os.path.join('./tmp/mosaic_mask',imagepath),0)
if size==0:
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_origin)
else:
mosaic_input = np.zeros((INPUT_SIZE,INPUT_SIZE,3*N+1), dtype='uint8')
for j in range(0,N):
img = impro.imread(os.path.join('./tmp/video2image',imagepaths[np.clip(i+j-12,0,len(imagepaths)-1)]))
img = img[y-size:y+size,x-size:x+size]
img = impro.resize(img,INPUT_SIZE)
mosaic_input[:,:,j*3:(j+1)*3] = img
mask = impro.resize(mask,np.min(img_origin.shape[:2]))
mask = mask[y-size:y+size,x-size:x+size]
mask = impro.resize(mask, INPUT_SIZE)
mosaic_input[:,:,-1] = mask
mosaic_input = data.im2tensor(mosaic_input,bgr2rgb=False,use_gpu=opt.use_gpu,use_transform = False,is0_1 = False)
unmosaic_pred = netG(mosaic_input)
#unmosaic_pred = (unmosaic_pred.cpu().detach().numpy()*255)[0]
#img_fake = unmosaic_pred.transpose((1, 2, 0))
img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = False ,is0_1 = False)
img_result = impro.replace_mosaic(img_origin,img_fake,x,y,size,opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic',imagepath),img_result)
print('\r','Clean Mosaic:'+str(i+1)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=40),end='')
print()
ffmpeg.image2video( fps,
'./tmp/replace_mosaic/output_%05d.'+opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4'))
def get_mosaic_positions(opt,netM,imagepaths,savemask=True):
# get mosaic position
positions = []
for i,imagepath in enumerate(imagepaths,1):
img_origin = impro.imread(os.path.join('./tmp/video2image',imagepath))
x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
if savemask:
cv2.imwrite(os.path.join('./tmp/mosaic_mask',imagepath), mask)
positions.append([x,y,size])
print('\r','Find mosaic location:'+str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=35),end='')
print('\nOptimize mosaic locations...')
positions =np.array(positions)
for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num)
return positions
def cleanmosaic_img(opt,netG,netM):
path = opt.media_path
print('Clean Mosaic:',path)
img_origin = impro.imread(path)
x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
#cv2.imwrite('./mask/'+os.path.basename(path), mask)
img_result = img_origin.copy()
if size != 0 :
img_mosaic = img_origin[y-size:y+size,x-size:x+size]
img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
img_result = impro.replace_mosaic(img_origin,img_fake,x,y,size,opt.no_feather)
else:
print('Do not find mosaic')
impro.imwrite(os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.jpg'),img_result)
def cleanmosaic_video_byframe(opt):
netG = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
x, y, size = runmodel.get_mosaic_position(img_origin, net_mosaic_pos,
opt)[:3]
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
# clean mosaic
for i, imagepath in enumerate(imagepaths, 0):
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_result)
print('Clean Mosaic:', imagepath)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
positions = []
img_ori_example = impro.imread(
os.path.join('./tmp/video2image', imagepaths[0]))
mask_avg = np.zeros((impro.resize(img_ori_example, 128)).shape[:2])
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
print('Find ROI location:', imagepath)
img = impro.imread(imagepath)
x, y, size, mask = runmodel.get_mosaic_position(img,
net,
opt,
threshold=64)
cv2.imwrite(
os.path.join('./tmp/ROI_mask', os.path.basename(imagepath)), mask)
positions.append([x, y, size])
mask_avg = mask_avg + mask
print('Optimize ROI locations...')
mask_index = filt.position_medfilt(np.array(positions), 13)
mask = np.clip(mask_avg / len(imagepaths), 0, 255).astype('uint8')
mask = impro.mask_threshold(mask, 20, 32)
x, y, size, area = impro.boundingSquare(mask, Ex_mul=1.5)
rat = min(img_ori_example.shape[:2]) / 128.0
x, y, size = int(rat * x), int(rat * y), int(rat * size)
cv2.imwrite(os.path.join('./tmp/ROI_mask_check', 'test_show.png'), mask)
if size != 0:
def cleanmosaic_video_fusion(opt):
net = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
N = 25
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
# x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3]
x, y, size, mask = runmodel.get_mosaic_position(
img_origin, net_mosaic_pos, opt)
cv2.imwrite(os.path.join('./tmp/mosaic_mask', imagepath), mask)
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
# clean mosaic
print('Clean mosaic...')
for i, imagepath in enumerate(imagepaths, 0):
print('Clean mosaic:', imagepath)
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
mask = cv2.imread(os.path.join('./tmp/mosaic_mask', imagepath), 0)
if size == 0:
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_origin)
else:
mosaic_input = np.zeros((256, 256, 3 * N + 1), dtype='uint8')
for j in range(0, N):
img = impro.imread(
os.path.join(
'./tmp/video2image',
imagepaths[np.clip(i + j - 12, 0,
len(imagepaths) - 1)]))
img = img[y - size:y + size, x - size:x + size]
img = impro.resize(img, 256)
mosaic_input[:, :, j * 3:(j + 1) * 3] = img
mask = impro.resize(mask, np.min(img_origin.shape[:2]))
mask = mask[y - size:y + size, x - size:x + size]
mask = impro.resize(mask, 256)
mosaic_input[:, :, -1] = mask
mosaic_input = data.im2tensor(mosaic_input,
bgr2rgb=False,
use_gpu=opt.use_gpu,
use_transform=False)
unmosaic_pred = net(mosaic_input)
unmosaic_pred = (unmosaic_pred.cpu().detach().numpy() * 255)[0]
img_fake = unmosaic_pred.transpose((1, 2, 0))
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_result)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def main():
if opt.mode == 'add':
net = loadmodel.unet(opt)
path = opt.media_path
if util.is_img(path):
print('Add Mosaic:', path)
img = impro.imread(path)
mask = runmodel.get_ROI_position(img, net, opt)[0]
img = mosaic.addmosaic(img, mask, opt)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(
path, './tmp/video2image/output_%05d.' + opt.tempimage_type)
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
positions = []
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
print('Find ROI location:', imagepath)
img = impro.imread(imagepath)
mask, x, y, area = runmodel.get_ROI_position(img, net, opt)
positions.append([x, y, area])
cv2.imwrite(
os.path.join('./tmp/ROI_mask',
os.path.basename(imagepath)), mask)
print('Optimize ROI locations...')
mask_index = filt.position_medfilt(np.array(positions), 7)
# add mosaic
print('Add mosaic to images...')
for i in range(len(imagepaths)):
mask_path = os.path.join('./tmp/ROI_mask',
imagepaths[mask_index[i]])
mask = impro.imread(mask_path)
img = impro.imread(
os.path.join('./tmp/video2image', imagepaths[i]))
img = mosaic.addmosaic(img, mask, opt)
cv2.imwrite(
os.path.join('./tmp/addmosaic_image',
os.path.basename(imagepaths[i])), img)
ffmpeg.image2video(
fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_add.mp4'))
elif opt.mode == 'clean':
netG = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
if util.is_img(path):
print('Clean Mosaic:', path)
img_origin = impro.imread(path)
x, y, size = runmodel.get_mosaic_position(img_origin,
net_mosaic_pos, opt)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x, y,
size, opt.no_feather)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img_result)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(
path, './tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
img_origin = impro.imread(imagepath)
x, y, size = runmodel.get_mosaic_position(
img_origin, net_mosaic_pos, opt)
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i],
opt.medfilt_num)
# clean mosaic
for i, imagepath in enumerate(imagepaths, 0):
imagepath = os.path.join('./tmp/video2image', imagepath)
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(imagepath)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size,
x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x,
y, size, opt.no_feather)
cv2.imwrite(
os.path.join('./tmp/replace_mosaic',
os.path.basename(imagepath)), img_result)
print('Clean Mosaic:', imagepath)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] +
'_clean.mp4'))
util.clean_tempfiles(tmp_init=False)
def get_mosaic_positions(opt, netM, imagepaths, savemask=True):
# resume
continue_flag = False
if os.path.isfile(os.path.join(opt.temp_dir, 'step.json')):
step = util.loadjson(os.path.join(opt.temp_dir, 'step.json'))
resume_frame = int(step['frame'])
if int(step['step']) > 2:
pre_positions = np.load(
os.path.join(opt.temp_dir, 'mosaic_positions.npy'))
return pre_positions
if int(step['step']) >= 2 and resume_frame > 0:
pre_positions = np.load(
os.path.join(opt.temp_dir, 'mosaic_positions.npy'))
continue_flag = True
imagepaths = imagepaths[resume_frame:]
positions = []
t1 = time.time()
if not opt.no_preview:
cv2.namedWindow('mosaic mask', cv2.WINDOW_NORMAL)
print('Step:2/4 -- Find mosaic location')
img_read_pool = Queue(4)
def loader(imagepaths):
for imagepath in imagepaths:
img_origin = impro.imread(
os.path.join(opt.temp_dir + '/video2image', imagepath))
img_read_pool.put(img_origin)
t = Thread(target=loader, args=(imagepaths, ))
t.setDaemon(True)
t.start()
for i, imagepath in enumerate(imagepaths, 1):
img_origin = img_read_pool.get()
x, y, size, mask = runmodel.get_mosaic_position(img_origin, netM, opt)
positions.append([x, y, size])
if savemask:
t = Thread(target=cv2.imwrite,
args=(
os.path.join(opt.temp_dir + '/mosaic_mask',
imagepath),
mask,
))
t.start()
if i % 1000 == 0:
save_positions = np.array(positions)
if continue_flag:
save_positions = np.concatenate(
(pre_positions, save_positions), axis=0)
np.save(os.path.join(opt.temp_dir, 'mosaic_positions.npy'),
save_positions)
step = {'step': 2, 'frame': i + resume_frame}
util.savejson(os.path.join(opt.temp_dir, 'step.json'), step)
#preview result and print
if not opt.no_preview:
cv2.imshow('mosaic mask', mask)
cv2.waitKey(1) & 0xFF
t2 = time.time()
print('\r',
str(i) + '/' + str(len(imagepaths)),
util.get_bar(100 * i / len(imagepaths), num=35),
util.counttime(t1, t2, i, len(imagepaths)),
end='')
if not opt.no_preview:
cv2.destroyAllWindows()
print('\nOptimize mosaic locations...')
positions = np.array(positions)
if continue_flag:
positions = np.concatenate((pre_positions, positions), axis=0)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
step = {'step': 3, 'frame': 0}
util.savejson(os.path.join(opt.temp_dir, 'step.json'), step)
np.save(os.path.join(opt.temp_dir, 'mosaic_positions.npy'), positions)
return positions
ffmpeg.image2video(
fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_add.mp4'))
elif opt.mode == 'clean':
netG = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
if util.is_img(path):
print('Clean Mosaic:', path)
img_origin = impro.imread(path)
x, y, size = runmodel.get_mosaic_position(img_origin, net_mosaic_pos,
opt)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic,
netG,
use_gpu=opt.use_gpu)
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img_result)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
