def test_run_vmafossexec_nonexist_dis_file(self): exe = ExternalProgram.vmafossexec cmd = "{exe} yuv420p 576 324 {ref} {dis} {model} > /dev/null 2>&1".format( exe=exe, ref=VmafConfig.test_resource_path("yuv", "src01_hrc00_576x324.yuv"), dis=VmafConfig.resource_path("yuv", "src01_hrc01_576x324_XXX.yuv"), model=VmafConfig.model_path("other_models", "vmaf_v0.6.0.json")) ret = subprocess.call(cmd, shell=True) self.assertEqual(ret, self.RC_ARGUMENT_ISSUE)
def setUp(self): self.raw_dataset_filepath = VmafConfig.resource_path( "dataset", "NFLX_dataset_public_raw.py") self.derived_dataset_path = VmafConfig.workdir_path( "test_derived_dataset.py") self.derived_dataset_path_pyc = VmafConfig.workdir_path( "test_derived_dataset.pyc")
def main(): dataset_filepaths = [ VmafConfig.resource_path('dataset', 'NFLX_dataset_public_raw_last4outliers.py'), VmafConfig.resource_path('dataset', 'VQEGHD3_dataset_raw.py'), ] # ============ sample results ================= subjective_model_classes = [ MaximumLikelihoodEstimationModel, MosModel, # MaximumLikelihoodEstimationDmosModel, # DmosModel, ] # plot_sample_results(dataset_filepaths, subjective_model_classes) # ============ plot trends ================= # ===== datasize growth ===== # run_datasize_growth(dataset_filepaths) # ===== corrpution growth ===== run_subject_corruption_growth(dataset_filepaths) # run_random_corruption_growth(dataset_filepaths) # run_subject_partial_corruption_growth(dataset_filepaths) # ===== random missing growth ===== # run_missing_growth(dataset_filepaths) # ===== synthetic data ===== # validate_with_synthetic_dataset() plt.show()
def validate_with_synthetic_dataset(): # use the dataset_filepath only for its dimensions and reference video mapping dataset_filepath = VmafConfig.resource_path('dataset', 'NFLX_dataset_public_raw_last4outliers.py') np.random.seed(0) _validate_with_synthetic_dataset( subjective_model_classes=[ MaximumLikelihoodEstimationModel ], dataset_filepath=dataset_filepath, synthetic_result={ 'quality_scores': np.random.uniform(1, 5, 79), 'observer_bias': np.random.normal(0, 1, 30), 'observer_inconsistency': np.abs(np.random.uniform(0.0, 0.4, 30)), 'content_bias': np.random.normal(0, 0.00001, 9), 'content_ambiguity': np.abs(np.random.uniform(0.4, 0.6, 9)), } )
def main(): subsamples = [1, 2, 4, 8, 16, 32, 64, 128] elapsed_times = [] pccs = [] for subsample in subsamples: elapsed_time, srcc, pcc, rmse = run_vmafossexec_with_subsample( VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py'), subsample) elapsed_times.append(elapsed_time) pccs.append(pcc) print("SRCC: {}, PCC: {}, RMSE: {}, time: {}".format(srcc, pcc, rmse, elapsed_time)) fig, ax = plt.subplots(1, 1, figsize=[8, 5]) ax.plot(subsamples, 6*24*79 / np.array(elapsed_times), 'x-') ax.set_xlabel("Subsample") ax.set_ylabel("Processing Speed (Frms/Sec)") ax.grid(True) plt.tight_layout() DisplayConfig.show()
dataset_name = 'example_image' yuv_fmt = 'notyuv' quality_width = 1920 quality_height = 1080 workfile_yuv_type = 'yuv444p' from vmaf.config import VmafConfig ref_videos = [ { 'content_id': 0, 'path': VmafConfig.resource_path('icpf', 'frame00000001.icpf') }, { 'content_id': 1, 'path': VmafConfig.resource_path('icpf', 'frame00000002.icpf') }, ] dis_videos = [ { 'content_id': 0, 'asset_id': 0, 'dmos': 100, 'path': VmafConfig.resource_path('icpf', 'frame00000000.icpf') }, # ref { 'content_id': 0, 'asset_id': 1, 'dmos': 50,
dataset_name = 'example' quality_width = 1920 quality_height = 1080 from vmaf.config import VmafConfig ref_videos = [ { 'content_id': 0, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv'), 'yuv_fmt': 'yuv420p' }, ] dis_videos = [ { 'content_id': 0, 'asset_id': 0, 'dmos': 100, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv'), 'fps': 30, 'rebuf_indices': [0, 4, 15]
from vmaf.config import VmafConfig from vmaf.core.executor import run_executors_in_parallel from vmaf.core.raw_extractor import DisYUVRawVideoExtractor from vmaf.core.nn_train_test_model import ToddNoiseClassifierTrainTestModel from vmaf.routine import read_dataset from vmaf.tools.misc import import_python_file # parameters num_train = 500 num_test = 50 n_epochs = 30 seed = 0 # None # read input dataset dataset_path = VmafConfig.resource_path('dataset', 'BSDS500_noisy_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) # shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = VmafConfig.workdir_path('rawvideo.hdf5') raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file( raw_video_h5py_filepath) print '======================== Extract raw YUVs ==============================' _, raw_yuvs = run_executors_in_parallel(
from vmaf.config import VmafConfig dataset_name = 'test_image' yuv_fmt = 'yuv444p' ref_videos = [ {'content_id': 0, 'content_name': '100007', 'height': 321, 'path': VmafConfig.resource_path('test_image_yuv', '100007.yuv'), 'width': 481}, {'content_id': 1, 'content_name': '100039', 'height': 321, 'path': VmafConfig.resource_path('test_image_yuv', '100039.yuv'), 'width': 481}, {'content_id': 2, 'content_name': '100075', 'height': 321, 'path': VmafConfig.resource_path('test_image_yuv', '100075.yuv'), 'width': 481}, {'content_id': 4, 'content_name': '100098', 'height': 321, 'path': VmafConfig.resource_path('test_image_yuv', '100098.yuv'), 'width': 481}, {'content_id': 5, 'content_name': '100099', 'height': 321, 'path': VmafConfig.resource_path('test_image_yuv', '100099.yuv'), 'width': 481},
dataset_name = 'example' quality_width = 1920 quality_height = 1080 from vmaf.config import VmafConfig ref_videos = [ { 'content_id': 0, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv'), 'yuv_fmt': 'yuv420p', 'width': 1920, 'height': 1080 }, { 'content_id': 1, 'path': VmafConfig.resource_path('yuv', 'flat_1920_1080_0.yuv'), 'yuv_fmt': 'yuv420p', 'width': 720, 'height': 480 }, ] dis_videos = [ {
__copyright__ = "Copyright 2016-2017, Netflix, Inc." __license__ = "Apache, Version 2.0" import matplotlib.pyplot as plt import numpy as np from vmaf.config import VmafConfig from vmaf.routine import run_vmaf_cv, run_vmaf_kfold_cv if __name__ == '__main__': # ==== Run simple cross validation: one training and one testing dataset ==== run_vmaf_cv( train_dataset_filepath=VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py'), test_dataset_filepath=VmafConfig.resource_path('dataset', 'VQEGHD3_dataset.py'), param_filepath=VmafConfig.resource_path('param', 'vmaf_v3.py'), output_model_filepath=VmafConfig.workspace_path('model', 'test_model1.pkl'), ) # ==== Run cross validation across genres (tough test) ==== nflx_dataset_path = VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py') contentid_groups = [ [0, 5], # cartoon: BigBuckBunny, FoxBird [1], # CG: BirdsInCage [2, 6, 7], # complex: CrowdRun, OldTownCross, Seeking [3, 4], # ElFuente: ElFuente1, ElFuente2 [8], # sports: Tennis ] param_filepath = VmafConfig.resource_path('param', 'vmaf_v3.py')
dataset_name = 'example' yuv_fmt = 'yuv420p' width = 576 height = 324 quality_width = 576 quality_height = 324 from vmaf.config import VmafConfig ref_videos = [ {'content_id': 0, 'path': VmafConfig.resource_path('yuv', 'src01_hrc00_576x324.yuv')}, ] dis_videos = [ {'content_id': 0, 'asset_id': 1, 'dmos': 50, 'path': VmafConfig.resource_path('yuv', 'src01_hrc01_576x324.yuv'), 'crop_cmd': '288:162:144:81'}, {'content_id': 0, 'asset_id': 2, 'dmos': 49, 'path': VmafConfig.resource_path('yuv', 'src01_hrc01_576x324.yuv'), 'pad_cmd': 'iw+100:ih+100:50:50'}, {'content_id': 0, 'asset_id': 3, 'dmos': 48, 'path': VmafConfig.resource_path('yuv', 'src01_hrc01_576x324.yuv'), 'crop_cmd': '288:162:144:81', 'pad_cmd': 'iw+288:ih+162:144:81',}, ]
__copyright__ = "Copyright 2016-2019, Netflix, Inc." __license__ = "Apache, Version 2.0" import matplotlib.pyplot as plt import numpy as np from vmaf.config import VmafConfig, DisplayConfig from vmaf.routine import run_vmaf_cv, run_vmaf_kfold_cv if __name__ == '__main__': # ==== Run simple cross validation: one training and one testing dataset ==== run_vmaf_cv( train_dataset_filepath=VmafConfig.resource_path( 'dataset', 'NFLX_dataset_public.py'), test_dataset_filepath=VmafConfig.resource_path('dataset', 'VQEGHD3_dataset.py'), param_filepath=VmafConfig.resource_path('param', 'vmaf_v3.py'), output_model_filepath=VmafConfig.workspace_path( 'model', 'test_model1.pkl'), ) # ==== Run cross validation across genres (tough test) ==== nflx_dataset_path = VmafConfig.resource_path('dataset', 'NFLX_dataset_public.py') contentid_groups = [ [0, 5], # cartoon: BigBuckBunny, FoxBird [1], # CG: BirdsInCage [2, 6, 7], # complex: CrowdRun, OldTownCross, Seeking
dataset_name = 'example' from vmaf.config import VmafConfig ref_videos = [ {'content_id': 0, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv'), 'width': 1920, 'height': 1080, 'yuv_fmt': 'yuv420p'}, {'content_id': 1, 'path': VmafConfig.resource_path('yuv', 'flat_1280_720_0.yuv'), 'width': 1280, 'height': 720, 'yuv_fmt': 'yuv420p10le'}, ] dis_videos = [ {'content_id': 0, 'asset_id': 0, 'groundtruth': 100, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_0_0.yuv')}, # ref {'content_id': 0, 'asset_id': 1, 'groundtruth': 50, 'path': VmafConfig.resource_path('yuv', 'checkerboard_1920_1080_10_3_1_0.yuv')}, {'content_id': 1, 'asset_id': 2, 'groundtruth': 100, 'path': VmafConfig.resource_path('yuv', 'flat_1280_720_0.yuv')}, # ref {'content_id': 1, 'asset_id': 3, 'groundtruth': 80, 'path': VmafConfig.resource_path('yuv', 'flat_1280_720_10.yuv')}, ]
from vmaf.config import VmafConfig dataset_name = 'test_image' yuv_fmt = 'yuv444p' dataset_dir = VmafConfig.resource_path('test_image_yuv') quality_width = 200 quality_height = 100 resampling_type = 'bicubic' ref_videos = [ {'content_id': 0, 'content_name': '100007', 'height': 321, 'path': dataset_dir + '/100007.yuv', 'width': 481}, {'content_id': 1, 'content_name': '100039', 'height': 321, 'path': dataset_dir + '/100039.yuv', 'width': 481}, {'content_id': 2, 'content_name': '100075', 'height': 321, 'path': dataset_dir + '/100075.yuv', 'width': 481}, {'content_id': 4, 'content_name': '100098', 'height': 321, 'path': dataset_dir + '/100098.yuv',
def main(): # parameters num_train = 500 num_test = 50 n_epochs = 30 seed = 0 # None # read input dataset dataset_path = VmafConfig.resource_path('dataset', 'BSDS500_noisy_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) # shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = VmafConfig.workdir_path('rawvideo.hdf5') raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file( raw_video_h5py_filepath) print( '======================== Extract raw YUVs ==============================' ) _, raw_yuvs = run_executors_in_parallel( DisYUVRawVideoExtractor, assets, fifo_mode=True, delete_workdir=True, parallelize=False, # CAN ONLY USE SERIAL MODE FOR DisYRawVideoExtractor result_store=None, optional_dict=None, optional_dict2={'h5py_file': raw_video_h5py_file}) patch_h5py_filepath = VmafConfig.workdir_path('patch.hdf5') patch_h5py_file = ToddNoiseClassifierTrainTestModel.open_h5py_file( patch_h5py_filepath) model = ToddNoiseClassifierTrainTestModel( param_dict={ 'seed': seed, 'n_epochs': n_epochs, }, logger=None, optional_dict2={ # for options that won't impact the result # 'checkpoints_dir': VmafConfig.workspace_path('checkpoints_dir'), 'h5py_file': patch_h5py_file, }) print( '============================ Train model ===============================' ) xys = ToddNoiseClassifierTrainTestModel.get_xys_from_results( raw_yuvs[:num_train]) model.train(xys) print( '=========================== Evaluate model =============================' ) xs = ToddNoiseClassifierTrainTestModel.get_xs_from_results( raw_yuvs[num_train:]) ys = ToddNoiseClassifierTrainTestModel.get_ys_from_results( raw_yuvs[num_train:]) result = model.evaluate(xs, ys) print("") print("f1 test %g, errorrate test %g" % (result['f1'], result['errorrate'])) # tear down DisYUVRawVideoExtractor.close_h5py_file(raw_video_h5py_file) ToddNoiseClassifierTrainTestModel.close_h5py_file(patch_h5py_file) os.remove(raw_video_h5py_filepath) os.remove(patch_h5py_filepath) print('Done.')
def setUp(self): self.raw_dataset_filepath = VmafConfig.resource_path("dataset", "NFLX_dataset_public_raw.py") self.derived_dataset_path = VmafConfig.workdir_path("test_derived_dataset.py") self.derived_dataset_path_pyc = VmafConfig.workdir_path("test_derived_dataset.pyc")
from vmaf.config import VmafConfig from vmaf.core.executor import run_executors_in_parallel from vmaf.core.raw_extractor import DisYUVRawVideoExtractor from vmaf.core.nn_train_test_model import ToddNoiseClassifierTrainTestModel from vmaf.routine import read_dataset from vmaf.tools.misc import import_python_file # parameters num_train = 500 num_test = 50 n_epochs = 30 seed = 0 # None # read input dataset dataset_path = VmafConfig.resource_path('dataset', 'BSDS500_noisy_dataset.py') dataset = import_python_file(dataset_path) assets = read_dataset(dataset) # shuffle assets np.random.seed(seed) np.random.shuffle(assets) assets = assets[:(num_train + num_test)] raw_video_h5py_filepath = VmafConfig.workdir_path('rawvideo.hdf5') raw_video_h5py_file = DisYUVRawVideoExtractor.open_h5py_file(raw_video_h5py_filepath) print '======================== Extract raw YUVs ==============================' _, raw_yuvs = run_executors_in_parallel( DisYUVRawVideoExtractor,