def process_video(videopath, savedir, min_interval_time=0.1, crop_mode='peak', crop_time=0.2, rate=44100, fc=[20,8000], saveimage=True): """ videopath: savedir: min_interval_time: crop_mode: 'peak' | 'time' crop_time: rate: saveimage: return: video_infos :fps,endtime,height,width peakindexs bias syllables """ util.makedirs(savedir) # process video video_infos = ffmpeg.get_video_infos(videopath) ffmpeg.video2voice(videopath, os.path.join(savedir,'video_tmp.wav'),samplingrate=44100) if saveimage: util.makedirs(os.path.join(savedir,'imgs')) ffmpeg.video2image(videopath,os.path.join(savedir,'imgs','%05d.png')) # process audio audio,syllables,features,peakindexs,bias = process_audio(os.path.join(savedir,'video_tmp.wav'), savedir, min_interval_time,crop_mode, crop_time, rate, fc) np.save(os.path.join(savedir,'video_infos.npy'), np.array(video_infos)) return audio,syllables,features,peakindexs,bias,video_infos
def video_init(opt, path): fps, endtime, height, width = ffmpeg.get_video_infos(path) if opt.fps != 0: fps = opt.fps continue_flag = False imagepaths = [] if os.path.isdir(opt.temp_dir): imagepaths = os.listdir(opt.temp_dir + '/video2image') if imagepaths != []: imagepaths.sort() last_frame = int(imagepaths[-1][7:13]) if (opt.last_time != '00:00:00' and last_frame > fps*(util.stamp2second(opt.last_time)-1)) \ or (opt.last_time == '00:00:00' and last_frame > fps*(endtime-1)): choose = input( 'There is an unfinished video. Continue it? [y/n] ') if choose.lower() == 'yes' or choose.lower() == 'y': continue_flag = True if not continue_flag: print('Step:1/4 -- Convert video to images') util.file_init(opt) ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3', opt.start_time, opt.last_time) ffmpeg.video2image( path, opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type, fps, opt.start_time, opt.last_time) imagepaths = os.listdir(opt.temp_dir + '/video2image') imagepaths.sort() return fps, imagepaths, height, width
def video_init(opt, path): fps, endtime, height, width = ffmpeg.get_video_infos(path) if opt.fps != 0: fps = opt.fps # resume if os.path.isfile(os.path.join(opt.temp_dir, 'step.json')): step = util.loadjson(os.path.join(opt.temp_dir, 'step.json')) if int(step['step']) >= 1: choose = input('There is an unfinished video. Continue it? [y/n] ') if choose.lower() == 'yes' or choose.lower() == 'y': imagepaths = os.listdir(opt.temp_dir + '/video2image') imagepaths.sort() return fps, imagepaths, height, width print('Step:1/4 -- Convert video to images') util.file_init(opt) ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3', opt.start_time, opt.last_time) ffmpeg.video2image( path, opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type, fps, opt.start_time, opt.last_time) imagepaths = os.listdir(opt.temp_dir + '/video2image') imagepaths.sort() step = {'step': 2, 'frame': 0} util.savejson(os.path.join(opt.temp_dir, 'step.json'), step) return fps, imagepaths, height, width
def video_init(opt,path): util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path,'./tmp/voice_tmp.mp3') ffmpeg.video2image(path,'./tmp/video2image/output_%05d.'+opt.tempimage_type) imagepaths=os.listdir('./tmp/video2image') imagepaths.sort() return fps,imagepaths
def video_init(opt,path): util.clean_tempfiles() fps,endtime,height,width = ffmpeg.get_video_infos(path) if opt.fps !=0: fps = opt.fps ffmpeg.video2voice(path,'./tmp/voice_tmp.mp3') ffmpeg.video2image(path,'./tmp/video2image/output_%05d.'+opt.tempimage_type,fps) imagepaths=os.listdir('./tmp/video2image') imagepaths.sort() return fps,imagepaths,height,width
def video_init(opt, path): util.clean_tempfiles(opt) fps, endtime, height, width = ffmpeg.get_video_infos(path) if opt.fps != 0: fps = opt.fps ffmpeg.video2voice(path, opt.temp_dir + '/voice_tmp.mp3', opt.start_time, opt.last_time) ffmpeg.video2image( path, opt.temp_dir + '/video2image/output_%06d.' + opt.tempimage_type, fps, opt.start_time, opt.last_time) imagepaths = os.listdir(opt.temp_dir + '/video2image') imagepaths.sort() return fps, imagepaths, height, width
def cleanmosaic_video_byframe(opt): netG = loadmodel.pix2pix(opt) net_mosaic_pos = loadmodel.unet_clean(opt) path = opt.media_path util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image(path, './tmp/video2image/output_%05d.' + opt.tempimage_type) positions = [] imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() # get position for imagepath in imagepaths: img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath)) x, y, size = runmodel.get_mosaic_position(img_origin, net_mosaic_pos, opt)[:3] positions.append([x, y, size]) print('Find mosaic location:', imagepath) print('Optimize mosaic locations...') positions = np.array(positions) for i in range(3): positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num) # clean mosaic for i, imagepath in enumerate(imagepaths, 0): x, y, size = positions[i][0], positions[i][1], positions[i][2] img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath)) img_result = img_origin.copy() if size != 0: img_mosaic = img_origin[y - size:y + size, x - size:x + size] img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt) img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size, opt.no_feather) cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath), img_result) print('Clean Mosaic:', imagepath) ffmpeg.image2video( fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type, './tmp/voice_tmp.mp3', os.path.join( opt.result_dir, os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def addmosaic_video(opt): net = loadmodel.unet(opt) path = opt.media_path util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image(path, './tmp/video2image/output_%05d.' + opt.tempimage_type) imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() # get position positions = [] for imagepath in imagepaths: print('Find ROI location:', imagepath) img = impro.imread(os.path.join('./tmp/video2image', imagepath)) mask, x, y, area = runmodel.get_ROI_position(img, net, opt) positions.append([x, y, area]) cv2.imwrite(os.path.join('./tmp/ROI_mask', imagepath), mask) print('Optimize ROI locations...') mask_index = filt.position_medfilt(np.array(positions), 7) # add mosaic print('Add mosaic to images...') for i in range(len(imagepaths)): mask = impro.imread( os.path.join('./tmp/ROI_mask', imagepaths[mask_index[i]])) img = impro.imread(os.path.join('./tmp/video2image', imagepaths[i])) img = mosaic.addmosaic(img, mask, opt) cv2.imwrite(os.path.join('./tmp/addmosaic_image', imagepaths[i]), img) ffmpeg.image2video( fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type, './tmp/voice_tmp.mp3', os.path.join(opt.result_dir, os.path.splitext(os.path.basename(path))[0] + '_add.mp4'))
videopaths = util.is_videos(videopaths) random.shuffle(videopaths) # def network net = loadmodel.bisenet(opt, 'roi') result_cnt = opt.startcnt video_cnt = 1 starttime = datetime.datetime.now() for videopath in videopaths: try: if opt.no_sclectscene: timestamps = ['00:00:00'] else: timestamps = [] fps, endtime, height, width = ffmpeg.get_video_infos(videopath) for cut_point in range(1, int( (endtime - opt.time) / opt.interval)): util.clean_tempfiles(opt) ffmpeg.video2image( videopath, opt.temp_dir + '/video2image/%05d.' + opt.tempimage_type, fps=1, start_time=util.second2stamp(cut_point * opt.interval), last_time=util.second2stamp(opt.time)) imagepaths = util.Traversal(opt.temp_dir + '/video2image') imagepaths = sorted(imagepaths) cnt = 0 for i in range(opt.time): img = impro.imread(imagepaths[i]) mask = runmodel.get_ROI_position(img,
print('Clean Mosaic:', path) img_origin = cv2.imread(path) x, y, size = get_mosaic_position(img_origin) img_result = img_origin.copy() if size != 0: img_mosaic = img_origin[y - size:y + size, x - size:x + size] img_fake = runmodel.run_pix2pix(img_mosaic, netG, use_gpu=opt.use_gpu) img_result = replace_mosaic(img_origin, img_fake, x, y, size) cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)), img_result) elif util.is_video(path): util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image( path, './tmp/video2image/output_%05d.' + opt.tempimage_type) positions = [] imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() for imagepath in imagepaths: imagepath = os.path.join('./tmp/video2image', imagepath) img_origin = cv2.imread(imagepath) x, y, size = get_mosaic_position(img_origin) positions.append([x, y, size]) print('Find Positions:', imagepath) positions = np.array(positions) for i in range(3):
def cleanmosaic_video_fusion(opt): net = loadmodel.pix2pix(opt) net_mosaic_pos = loadmodel.unet_clean(opt) path = opt.media_path N = 25 util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image(path, './tmp/video2image/output_%05d.' + opt.tempimage_type) positions = [] imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() # get position for imagepath in imagepaths: img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath)) # x,y,size = runmodel.get_mosaic_position(img_origin,net_mosaic_pos,opt)[:3] x, y, size, mask = runmodel.get_mosaic_position( img_origin, net_mosaic_pos, opt) cv2.imwrite(os.path.join('./tmp/mosaic_mask', imagepath), mask) positions.append([x, y, size]) print('Find mosaic location:', imagepath) print('Optimize mosaic locations...') positions = np.array(positions) for i in range(3): positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num) # clean mosaic print('Clean mosaic...') for i, imagepath in enumerate(imagepaths, 0): print('Clean mosaic:', imagepath) x, y, size = positions[i][0], positions[i][1], positions[i][2] img_origin = impro.imread(os.path.join('./tmp/video2image', imagepath)) mask = cv2.imread(os.path.join('./tmp/mosaic_mask', imagepath), 0) if size == 0: cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath), img_origin) else: mosaic_input = np.zeros((256, 256, 3 * N + 1), dtype='uint8') for j in range(0, N): img = impro.imread( os.path.join( './tmp/video2image', imagepaths[np.clip(i + j - 12, 0, len(imagepaths) - 1)])) img = img[y - size:y + size, x - size:x + size] img = impro.resize(img, 256) mosaic_input[:, :, j * 3:(j + 1) * 3] = img mask = impro.resize(mask, np.min(img_origin.shape[:2])) mask = mask[y - size:y + size, x - size:x + size] mask = impro.resize(mask, 256) mosaic_input[:, :, -1] = mask mosaic_input = data.im2tensor(mosaic_input, bgr2rgb=False, use_gpu=opt.use_gpu, use_transform=False) unmosaic_pred = net(mosaic_input) unmosaic_pred = (unmosaic_pred.cpu().detach().numpy() * 255)[0] img_fake = unmosaic_pred.transpose((1, 2, 0)) img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size, opt.no_feather) cv2.imwrite(os.path.join('./tmp/replace_mosaic', imagepath), img_result) ffmpeg.image2video( fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type, './tmp/voice_tmp.mp3', os.path.join( opt.result_dir, os.path.splitext(os.path.basename(path))[0] + '_clean.mp4'))
def main(): if opt.mode == 'add': net = loadmodel.unet(opt) path = opt.media_path if util.is_img(path): print('Add Mosaic:', path) img = impro.imread(path) mask = runmodel.get_ROI_position(img, net, opt)[0] img = mosaic.addmosaic(img, mask, opt) cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)), img) elif util.is_video(path): util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image( path, './tmp/video2image/output_%05d.' + opt.tempimage_type) imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() # get position positions = [] for imagepath in imagepaths: imagepath = os.path.join('./tmp/video2image', imagepath) print('Find ROI location:', imagepath) img = impro.imread(imagepath) mask, x, y, area = runmodel.get_ROI_position(img, net, opt) positions.append([x, y, area]) cv2.imwrite( os.path.join('./tmp/ROI_mask', os.path.basename(imagepath)), mask) print('Optimize ROI locations...') mask_index = filt.position_medfilt(np.array(positions), 7) # add mosaic print('Add mosaic to images...') for i in range(len(imagepaths)): mask_path = os.path.join('./tmp/ROI_mask', imagepaths[mask_index[i]]) mask = impro.imread(mask_path) img = impro.imread( os.path.join('./tmp/video2image', imagepaths[i])) img = mosaic.addmosaic(img, mask, opt) cv2.imwrite( os.path.join('./tmp/addmosaic_image', os.path.basename(imagepaths[i])), img) ffmpeg.image2video( fps, './tmp/addmosaic_image/output_%05d.' + opt.tempimage_type, './tmp/voice_tmp.mp3', os.path.join( opt.result_dir, os.path.splitext(os.path.basename(path))[0] + '_add.mp4')) elif opt.mode == 'clean': netG = loadmodel.pix2pix(opt) net_mosaic_pos = loadmodel.unet_clean(opt) path = opt.media_path if util.is_img(path): print('Clean Mosaic:', path) img_origin = impro.imread(path) x, y, size = runmodel.get_mosaic_position(img_origin, net_mosaic_pos, opt) img_result = img_origin.copy() if size != 0: img_mosaic = img_origin[y - size:y + size, x - size:x + size] img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt) img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size, opt.no_feather) cv2.imwrite(os.path.join(opt.result_dir, os.path.basename(path)), img_result) elif util.is_video(path): util.clean_tempfiles() fps = ffmpeg.get_video_infos(path)[0] ffmpeg.video2voice(path, './tmp/voice_tmp.mp3') ffmpeg.video2image( path, './tmp/video2image/output_%05d.' + opt.tempimage_type) positions = [] imagepaths = os.listdir('./tmp/video2image') imagepaths.sort() # get position for imagepath in imagepaths: imagepath = os.path.join('./tmp/video2image', imagepath) img_origin = impro.imread(imagepath) x, y, size = runmodel.get_mosaic_position( img_origin, net_mosaic_pos, opt) positions.append([x, y, size]) print('Find mosaic location:', imagepath) print('Optimize mosaic locations...') positions = np.array(positions) for i in range(3): positions[:, i] = filt.medfilt(positions[:, i], opt.medfilt_num) # clean mosaic for i, imagepath in enumerate(imagepaths, 0): imagepath = os.path.join('./tmp/video2image', imagepath) x, y, size = positions[i][0], positions[i][1], positions[i][2] img_origin = impro.imread(imagepath) img_result = img_origin.copy() if size != 0: img_mosaic = img_origin[y - size:y + size, x - size:x + size] img_fake = runmodel.run_pix2pix(img_mosaic, netG, opt) img_result = impro.replace_mosaic(img_origin, img_fake, x, y, size, opt.no_feather) cv2.imwrite( os.path.join('./tmp/replace_mosaic', os.path.basename(imagepath)), img_result) print('Clean Mosaic:', imagepath) ffmpeg.image2video( fps, './tmp/replace_mosaic/output_%05d.' + opt.tempimage_type, './tmp/voice_tmp.mp3', os.path.join( opt.result_dir, os.path.splitext(os.path.basename(path))[0] + '_clean.mp4')) util.clean_tempfiles(tmp_init=False)
util.clean_tempfiles(tmp_init=False) util.makedirs('./tmp/voice') seed_voices = [] seed_freqs = [] for i in range(len(video_names)): ffmpeg.video2voice(os.path.join(dataset, video_names[i]), os.path.join('./tmp/voice', '%03d' % i + '.wav'), samplingrate=44100) voice = sound.load(os.path.join('./tmp/voice', '%03d' % i + '.wav'))[1] #voice = dsp.bpf(voice, 44100, 20, 2000) base_freq = sound.basefreq(voice, 44100, 4000, mode='mean') seed_voices.append(voice) seed_freqs.append(base_freq) fps, endtime, height, width = ffmpeg.get_video_infos( os.path.join(dataset, video_names[i])) util.makedirs(os.path.join('./tmp/video2image', '%03d' % i)) ffmpeg.video2image( os.path.join(dataset, video_names[i]), os.path.join('./tmp/video2image', '%03d' % i, '%05d.jpg')) print('Generating voice...') sinmusic, musicinfos = notation.notations2music(notations, mode='sin') music = np.zeros_like(sinmusic) for i in range(len(musicinfos['time'])): for j in range(len(musicinfos['freq'][i])): if musicinfos['freq'][i][j] != 0: diff = np.abs( librosa.hz_to_octs(seed_freqs) - librosa.hz_to_octs(musicinfos['freq'][i][j]))
while True: t = 1.0/opt.fps time.sleep(t) timerQueueime.put(True) opt = Options().getparse() system_type = 'Linux' if 'Windows' in platform.platform(): system_type = 'Windows' #-------------------------------Media Init------------------------------- if util.is_img(opt.media): img = cv2.imread(opt.media) h_media,w_media = img.shape[:2] elif util.is_video(opt.media): fps,endtime,h_media,w_media = ffmpeg.get_video_infos(opt.media) if opt.frame_num == 0: opt.frame_num = int(endtime*fps-5) if opt.ori_fps == 0: opt.ori_fps = fps util.makedirs('./tmp') else: print('Can not load this file!') #-------------------------------Image Shape Init------------------------------- if opt.screen==1: limw = 80;limh = 24 if opt.screen==2: limw = 132;limh = 43 if opt.screen==3: limw = 203;limh = 55