def process_video(videopath, savedir, min_interval_time=0.1, crop_mode='peak', crop_time=0.2, rate=44100, fc=[20,8000], saveimage=True):
"""
videopath:
savedir:
min_interval_time:
crop_mode: 'peak' | 'time'
crop_time:
rate:
saveimage:
return:
video_infos :fps,endtime,height,width
peakindexs
bias
syllables
"""
util.makedirs(savedir)
# process video
video_infos = ffmpeg.get_video_infos(videopath)
ffmpeg.video2voice(videopath, os.path.join(savedir,'video_tmp.wav'),samplingrate=44100)
if saveimage:
util.makedirs(os.path.join(savedir,'imgs'))
ffmpeg.video2image(videopath,os.path.join(savedir,'imgs','%05d.png'))
# process audio
audio,syllables,features,peakindexs,bias = process_audio(os.path.join(savedir,'video_tmp.wav'),
savedir, min_interval_time,crop_mode, crop_time, rate, fc)
np.save(os.path.join(savedir,'video_infos.npy'), np.array(video_infos))
return audio,syllables,features,peakindexs,bias,video_infos
def video_init(opt, path):
fps, endtime, height, width = ffmpeg.get_video_infos(path)
if opt.fps != 0:
fps = opt.fps
continue_flag = False
imagepaths = []
if os.path.isdir(opt.temp_dir):
imagepaths = os.listdir(opt.temp_dir + '/video2image')
if imagepaths != []:
imagepaths.sort()
last_frame = int(imagepaths[-1][7:13])
if (opt.last_time != '00:00:00' and last_frame > fps*(util.stamp2second(opt.last_time)-1)) \
or (opt.last_time == '00:00:00' and last_frame > fps*(endtime-1)):
choose = input(
'There is an unfinished video. Continue it? [y/n] ')
if choose.lower() == 'yes' or choose.lower() == 'y':
continue_flag = True
if not continue_flag:
print('Step:1/4 -- Convert video to images')
util.file_init(opt)
ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3',
opt.start_time, opt.last_time)
ffmpeg.video2image(
path,
opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type,
fps, opt.start_time, opt.last_time)
imagepaths = os.listdir(opt.temp_dir + '/video2image')
imagepaths.sort()
return fps, imagepaths, height, width
def video_init(opt, path):
fps, endtime, height, width = ffmpeg.get_video_infos(path)
if opt.fps != 0:
fps = opt.fps
# resume
if os.path.isfile(os.path.join(opt.temp_dir, 'step.json')):
step = util.loadjson(os.path.join(opt.temp_dir, 'step.json'))
if int(step['step']) >= 1:
choose = input('There is an unfinished video. Continue it? [y/n] ')
if choose.lower() == 'yes' or choose.lower() == 'y':
imagepaths = os.listdir(opt.temp_dir + '/video2image')
imagepaths.sort()
return fps, imagepaths, height, width
print('Step:1/4 -- Convert video to images')
util.file_init(opt)
ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3', opt.start_time,
opt.last_time)
ffmpeg.video2image(
path, opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type,
fps, opt.start_time, opt.last_time)
imagepaths = os.listdir(opt.temp_dir + '/video2image')
imagepaths.sort()
step = {'step': 2, 'frame': 0}
util.savejson(os.path.join(opt.temp_dir, 'step.json'), step)
return fps, imagepaths, height, width
def video_init(opt,path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path,'./tmp/voice_tmp.mp3')
ffmpeg.video2image(path,'./tmp/video2image/output_%05d.'+opt.tempimage_type)
imagepaths=os.listdir('./tmp/video2image')
imagepaths.sort()
return fps,imagepaths
def video_init(opt,path):
util.clean_tempfiles()
fps,endtime,height,width = ffmpeg.get_video_infos(path)
if opt.fps !=0:
fps = opt.fps
ffmpeg.video2voice(path,'./tmp/voice_tmp.mp3')
ffmpeg.video2image(path,'./tmp/video2image/output_%05d.'+opt.tempimage_type,fps)
imagepaths=os.listdir('./tmp/video2image')
imagepaths.sort()
return fps,imagepaths,height,width
def video_init(opt, path):
util.clean_tempfiles(opt)
fps, endtime, height, width = ffmpeg.get_video_infos(path)
if opt.fps != 0:
fps = opt.fps
ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3', opt.start_time,
opt.last_time)
ffmpeg.video2image(
path, opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type,
fps, opt.start_time, opt.last_time)
imagepaths = os.listdir(opt.temp_dir + '/video2image')
imagepaths.sort()
return fps, imagepaths, height, width
def cleanmosaic_video_byframe(opt):
netG = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
x, y, size = runmodel.get_mosaic_position(img_origin, net_mosaic_pos,
opt)[:3]
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
# clean mosaic
for i, imagepath in enumerate(imagepaths, 0):
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_result)
print('Clean Mosaic:', imagepath)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def addmosaic_video(opt):
net = loadmodel.unet(opt)
path = opt.media_path
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
positions = []
for imagepath in imagepaths:
print('Find ROI location:', imagepath)
img = impro.imread(os.path.join('./tmp/video2image', imagepath))
mask, x, y, area = runmodel.get_ROI_position(img, net, opt)
positions.append([x, y, area])
cv2.imwrite(os.path.join('./tmp/ROI_mask', imagepath), mask)
print('Optimize ROI locations...')
mask_index = filt.position_medfilt(np.array(positions), 7)
# add mosaic
print('Add mosaic to images...')
for i in range(len(imagepaths)):
mask = impro.imread(
os.path.join('./tmp/ROI_mask', imagepaths[mask_index[i]]))
img = impro.imread(os.path.join('./tmp/video2image', imagepaths[i]))
img = mosaic.addmosaic(img, mask, opt)
cv2.imwrite(os.path.join('./tmp/addmosaic_image', imagepaths[i]), img)
ffmpeg.image2video(
fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_add.mp4'))
videopaths = util.is_videos(videopaths)
random.shuffle(videopaths)
# def network
net = loadmodel.bisenet(opt, 'roi')
result_cnt = opt.startcnt
video_cnt = 1
starttime = datetime.datetime.now()
for videopath in videopaths:
try:
if opt.no_sclectscene:
timestamps = ['00:00:00']
else:
timestamps = []
fps, endtime, height, width = ffmpeg.get_video_infos(videopath)
for cut_point in range(1, int(
(endtime - opt.time) / opt.interval)):
util.clean_tempfiles(opt)
ffmpeg.video2image(
videopath,
opt.temp_dir + '/video2image/%05d.' + opt.tempimage_type,
fps=1,
start_time=util.second2stamp(cut_point * opt.interval),
last_time=util.second2stamp(opt.time))
imagepaths = util.Traversal(opt.temp_dir + '/video2image')
imagepaths = sorted(imagepaths)
cnt = 0
for i in range(opt.time):
img = impro.imread(imagepaths[i])
mask = runmodel.get_ROI_position(img,
print('Clean Mosaic:', path)
img_origin = cv2.imread(path)
x, y, size = get_mosaic_position(img_origin)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic,
netG,
use_gpu=opt.use_gpu)
img_result = replace_mosaic(img_origin, img_fake, x, y, size)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img_result)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(
path, './tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
img_origin = cv2.imread(imagepath)
x, y, size = get_mosaic_position(img_origin)
positions.append([x, y, size])
print('Find Positions:', imagepath)
positions = np.array(positions)
for i in range(3):
def cleanmosaic_video_fusion(opt):
net = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
N = 25
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(path,
'./tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
# x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3]
x, y, size, mask = runmodel.get_mosaic_position(
img_origin, net_mosaic_pos, opt)
cv2.imwrite(os.path.join('./tmp/mosaic_mask', imagepath), mask)
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num)
# clean mosaic
print('Clean mosaic...')
for i, imagepath in enumerate(imagepaths, 0):
print('Clean mosaic:', imagepath)
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath))
mask = cv2.imread(os.path.join('./tmp/mosaic_mask', imagepath), 0)
if size == 0:
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_origin)
else:
mosaic_input = np.zeros((256, 256, 3 * N + 1), dtype='uint8')
for j in range(0, N):
img = impro.imread(
os.path.join(
'./tmp/video2image',
imagepaths[np.clip(i + j - 12, 0,
len(imagepaths) - 1)]))
img = img[y - size:y + size, x - size:x + size]
img = impro.resize(img, 256)
mosaic_input[:, :, j * 3:(j + 1) * 3] = img
mask = impro.resize(mask, np.min(img_origin.shape[:2]))
mask = mask[y - size:y + size, x - size:x + size]
mask = impro.resize(mask, 256)
mosaic_input[:, :, -1] = mask
mosaic_input = data.im2tensor(mosaic_input,
bgr2rgb=False,
use_gpu=opt.use_gpu,
use_transform=False)
unmosaic_pred = net(mosaic_input)
unmosaic_pred = (unmosaic_pred.cpu().detach().numpy() * 255)[0]
img_fake = unmosaic_pred.transpose((1, 2, 0))
img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size,
opt.no_feather)
cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath),
img_result)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def main():
if opt.mode == 'add':
net = loadmodel.unet(opt)
path = opt.media_path
if util.is_img(path):
print('Add Mosaic:', path)
img = impro.imread(path)
mask = runmodel.get_ROI_position(img, net, opt)[0]
img = mosaic.addmosaic(img, mask, opt)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(
path, './tmp/video2image/output_%05d.' + opt.tempimage_type)
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
positions = []
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
print('Find ROI location:', imagepath)
img = impro.imread(imagepath)
mask, x, y, area = runmodel.get_ROI_position(img, net, opt)
positions.append([x, y, area])
cv2.imwrite(
os.path.join('./tmp/ROI_mask',
os.path.basename(imagepath)), mask)
print('Optimize ROI locations...')
mask_index = filt.position_medfilt(np.array(positions), 7)
# add mosaic
print('Add mosaic to images...')
for i in range(len(imagepaths)):
mask_path = os.path.join('./tmp/ROI_mask',
imagepaths[mask_index[i]])
mask = impro.imread(mask_path)
img = impro.imread(
os.path.join('./tmp/video2image', imagepaths[i]))
img = mosaic.addmosaic(img, mask, opt)
cv2.imwrite(
os.path.join('./tmp/addmosaic_image',
os.path.basename(imagepaths[i])), img)
ffmpeg.image2video(
fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] + '_add.mp4'))
elif opt.mode == 'clean':
netG = loadmodel.pix2pix(opt)
net_mosaic_pos = loadmodel.unet_clean(opt)
path = opt.media_path
if util.is_img(path):
print('Clean Mosaic:', path)
img_origin = impro.imread(path)
x, y, size = runmodel.get_mosaic_position(img_origin,
net_mosaic_pos, opt)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size, x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x, y,
size, opt.no_feather)
cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)),
img_result)
elif util.is_video(path):
util.clean_tempfiles()
fps = ffmpeg.get_video_infos(path)[0]
ffmpeg.video2voice(path, './tmp/voice_tmp.mp3')
ffmpeg.video2image(
path, './tmp/video2image/output_%05d.' + opt.tempimage_type)
positions = []
imagepaths = os.listdir('./tmp/video2image')
imagepaths.sort()
# get position
for imagepath in imagepaths:
imagepath = os.path.join('./tmp/video2image', imagepath)
img_origin = impro.imread(imagepath)
x, y, size = runmodel.get_mosaic_position(
img_origin, net_mosaic_pos, opt)
positions.append([x, y, size])
print('Find mosaic location:', imagepath)
print('Optimize mosaic locations...')
positions = np.array(positions)
for i in range(3):
positions[:, i] = filt.medfilt(positions[:, i],
opt.medfilt_num)
# clean mosaic
for i, imagepath in enumerate(imagepaths, 0):
imagepath = os.path.join('./tmp/video2image', imagepath)
x, y, size = positions[i][0], positions[i][1], positions[i][2]
img_origin = impro.imread(imagepath)
img_result = img_origin.copy()
if size != 0:
img_mosaic = img_origin[y - size:y + size,
x - size:x + size]
img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt)
img_result = impro.replace_mosaic(img_origin, img_fake, x,
y, size, opt.no_feather)
cv2.imwrite(
os.path.join('./tmp/replace_mosaic',
os.path.basename(imagepath)), img_result)
print('Clean Mosaic:', imagepath)
ffmpeg.image2video(
fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type,
'./tmp/voice_tmp.mp3',
os.path.join(
opt.result_dir,
os.path.splitext(os.path.basename(path))[0] +
'_clean.mp4'))
util.clean_tempfiles(tmp_init=False)
util.clean_tempfiles(tmp_init=False)
util.makedirs('./tmp/voice')
seed_voices = []
seed_freqs = []
for i in range(len(video_names)):
ffmpeg.video2voice(os.path.join(dataset, video_names[i]),
os.path.join('./tmp/voice', '%03d' % i + '.wav'),
samplingrate=44100)
voice = sound.load(os.path.join('./tmp/voice', '%03d' % i + '.wav'))[1]
#voice = dsp.bpf(voice, 44100, 20, 2000)
base_freq = sound.basefreq(voice, 44100, 4000, mode='mean')
seed_voices.append(voice)
seed_freqs.append(base_freq)
fps, endtime, height, width = ffmpeg.get_video_infos(
os.path.join(dataset, video_names[i]))
util.makedirs(os.path.join('./tmp/video2image', '%03d' % i))
ffmpeg.video2image(
os.path.join(dataset, video_names[i]),
os.path.join('./tmp/video2image', '%03d' % i, '%05d.jpg'))
print('Generating voice...')
sinmusic, musicinfos = notation.notations2music(notations, mode='sin')
music = np.zeros_like(sinmusic)
for i in range(len(musicinfos['time'])):
for j in range(len(musicinfos['freq'][i])):
if musicinfos['freq'][i][j] != 0:
diff = np.abs(
librosa.hz_to_octs(seed_freqs) -
librosa.hz_to_octs(musicinfos['freq'][i][j]))
while True:
t = 1.0/opt.fps
time.sleep(t)
timerQueueime.put(True)
opt = Options().getparse()
system_type = 'Linux'
if 'Windows' in platform.platform():
system_type = 'Windows'
#-------------------------------Media Init-------------------------------
if util.is_img(opt.media):
img = cv2.imread(opt.media)
h_media,w_media = img.shape[:2]
elif util.is_video(opt.media):
fps,endtime,h_media,w_media = ffmpeg.get_video_infos(opt.media)
if opt.frame_num == 0:
opt.frame_num = int(endtime*fps-5)
if opt.ori_fps == 0:
opt.ori_fps = fps
util.makedirs('./tmp')
else:
print('Can not load this file!')
#-------------------------------Image Shape Init-------------------------------
if opt.screen==1:
limw = 80;limh = 24
if opt.screen==2:
limw = 132;limh = 43
if opt.screen==3:
limw = 203;limh = 55