def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature': ['vif', 'motion']},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
def cv_on_dataset(dataset,
feature_param,
model_param,
ax,
result_store,
contentid_groups,
logger=None,
aggregate_method=np.mean):
assets = read_dataset(dataset)
kfold = construct_kfold_list(assets, contentid_groups)
fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=assets,
logger=logger,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
fifo_mode=True,
# parallelize=False, fifo_mode=False, # VQM
)
fassembler.run()
results = fassembler.results
for result in results:
result.set_score_aggregate_method(aggregate_method)
model_class = TrainTestModel.find_subclass(model_param.model_type)
# run nested kfold cv for each combintation
cv_output = ModelCrossValidation.run_kfold_cross_validation(
model_class,
model_param.model_param_dict,
results,
kfold,
logger=logger,
)
print 'Feature parameters: {}'.format(feature_param.feature_dict)
print 'Model type: {}'.format(model_param.model_type)
print 'Model parameters: {}'.format(model_param.model_param_dict)
print 'Stats: {}'.format(model_class.format_stats(cv_output['aggr_stats']))
if ax is not None:
model_class.plot_scatter(ax, cv_output['aggr_stats'],
cv_output['contentids'])
ax.set_xlabel('True Score')
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title("Dataset: {dataset}, Model: {model},\n{stats}".format(
dataset=dataset.dataset_name,
model=model_param.model_type,
stats=model_class.format_stats(cv_output['aggr_stats'])))
return assets, cv_output
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature': 'all'},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
30.030914145833322)
def cv_on_dataset(
dataset, feature_param, model_param, ax, result_store, contentid_groups, logger=None, aggregate_method=np.mean
):
assets = read_dataset(dataset)
kfold = construct_kfold_list(assets, contentid_groups)
fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=assets,
logger=logger,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
fifo_mode=True,
# parallelize=False, fifo_mode=False, # VQM
)
fassembler.run()
results = fassembler.results
for result in results:
result.set_score_aggregate_method(aggregate_method)
model_class = TrainTestModel.find_subclass(model_param.model_type)
# run nested kfold cv for each combintation
cv_output = ModelCrossValidation.run_kfold_cross_validation(
model_class, model_param.model_param_dict, results, kfold, logger=logger
)
print "Feature parameters: {}".format(feature_param.feature_dict)
print "Model type: {}".format(model_param.model_type)
print "Model parameters: {}".format(model_param.model_param_dict)
print "Stats: {}".format(model_class.format_stats(cv_output["aggr_stats"]))
if ax is not None:
model_class.plot_scatter(ax, cv_output["aggr_stats"], cv_output["contentids"])
ax.set_xlabel("True Score")
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title(
"Dataset: {dataset}, Model: {model},\n{stats}".format(
dataset=dataset.dataset_name,
model=model_param.model_type,
stats=model_class.format_stats(cv_output["aggr_stats"]),
)
)
return assets, cv_output
def explain_model_on_dataset(model, test_assets_selected_indexs,
test_dataset_filepath):
def print_assets(test_assets):
print '\n'.join(
map(
lambda (i, asset): "Asset {i}: {name}".format(
i=i, name=get_file_name_without_extension(asset.dis_path)),
enumerate(test_assets)))
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print "Assets selected for local explanation: {}".format(
test_assets_selected_indexs)
result_store = FileSystemResultStore()
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict['feature_dict'],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps,
assets=test_assets,
ys=test_ys,
ys_pred=test_ys_pred)
plt.show()
def _get_vmaf_feature_assembler_instance(self, asset):
vmaf_fassembler = FeatureAssembler(
feature_dict=self.FEATURE_ASSEMBLER_DICT,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store)
return vmaf_fassembler
def explain_model_on_dataset(model, test_assets_selected_indexs, test_dataset_filepath):
def print_assets(test_assets):
print "\n".join(
map(
lambda (i, asset): "Asset {i}: {name}".format(
i=i, name=get_file_name_without_extension(asset.dis_path)
),
enumerate(test_assets),
)
)
test_dataset = import_python_file(test_dataset_filepath)
test_assets = read_dataset(test_dataset)
print_assets(test_assets)
print "Assets selected for local explanation: {}".format(test_assets_selected_indexs)
result_store = FileSystemResultStore()
test_assets = [test_assets[i] for i in test_assets_selected_indexs]
test_fassembler = FeatureAssembler(
feature_dict=model.model_dict["feature_dict"],
feature_option_dict=None,
assets=test_assets,
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_feature_results = test_fassembler.results
test_xs = model.get_xs_from_results(test_feature_results)
test_ys = model.get_ys_from_results(test_feature_results)
test_ys_pred = model.predict(test_xs)
explainer = LocalExplainer(neighbor_samples=1000)
test_exps = explainer.explain(model, test_xs)
explainer.print_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred)
explainer.plot_explanations(test_exps, assets=test_assets, ys=test_ys, ys_pred=test_ys_pred)
plt.show()
def _get_vmaf_feature_assembler_instance(self, asset):
vmaf_fassembler = FeatureAssembler(
feature_dict=self.FEATURE_ASSEMBLER_DICT,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return vmaf_fassembler
def _get_vmaf_feature_assembler_instance(self, asset):
# load TrainTestModel only to retrieve its 'feature_dict' extra info
model = self._load_model()
feature_dict = model.get_appended_info('feature_dict')
if feature_dict is None:
feature_dict = self.DEFAULT_FEATURE_DICT
vmaf_fassembler = FeatureAssembler(feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store)
return vmaf_fassembler
def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict = {'VMAF_feature':['vif', 'motion']},
feature_option_dict = None,
assets = [asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
def _get_feature_assembler_instance(self, asset):
feature_dict = {MsSsimFeatureExtractor.TYPE: MsSsimFeatureExtractor.ATOM_FEATURES}
feature_assembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return feature_assembler
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict = {'VMAF_feature':'all'},
feature_option_dict = None,
assets = [asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44609306249999997, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 4.0498253541666669, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141, places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 23.509571520833333, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 4.0498253541666669 , places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0, places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 31.271439270833337, places=4)
def _get_vmaf_feature_assembler_instance(self, asset):
# load TrainTestModel only to retrieve its 'feature_dict' extra info
feature_dict = self._load_model(asset).get_appended_info('feature_dict')
if feature_dict is None:
feature_dict = self.DEFAULT_FEATURE_DICT
vmaf_fassembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return vmaf_fassembler
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict = {'VMAF_feature':'all'},
feature_option_dict = None,
assets = [asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def _get_feature_assembler_instance(self, asset):
feature_dict = {
StrredFeatureExtractor.TYPE:
StrredFeatureExtractor.ATOM_FEATURES +
(StrredFeatureExtractor.DERIVED_ATOM_FEATURES if hasattr(
StrredFeatureExtractor, 'DERIVED_ATOM_FEATURES') else [])
}
feature_assembler = FeatureAssembler(
feature_dict=feature_dict,
feature_option_dict=None,
assets=[asset],
logger=self.logger,
fifo_mode=self.fifo_mode,
delete_workdir=self.delete_workdir,
result_store=self.result_store,
optional_dict=None,
optional_dict2=None,
parallelize=False, # parallelization already in a higher level
)
return feature_assembler
def train_test_vmaf_on_dataset(train_dataset, test_dataset,
feature_param, model_param,
train_ax, test_ax, result_store,
parallelize=True, logger=None, fifo_mode=True,
output_model_filepath=None,
aggregate_method=np.mean,
**kwargs):
train_assets = read_dataset(train_dataset, **kwargs)
train_raw_assets = None
try:
for train_asset in train_assets:
assert train_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = kwargs['subj_model_class'] if 'subj_model_class' in kwargs and kwargs['subj_model_class'] is not None else DmosModel
subjective_model = subj_model_class(RawDatasetReader(train_dataset))
subjective_model.run_modeling(**kwargs)
train_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs)
train_raw_assets = train_assets
train_assets = read_dataset(train_dataset_aggregate, **kwargs)
train_fassembler = FeatureAssembler(
feature_dict = feature_param.feature_dict,
feature_option_dict = None,
assets = train_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=parallelize,
)
train_fassembler.run()
train_features = train_fassembler.results
for result in train_features:
result.set_score_aggregate_method(aggregate_method)
model_type = model_param.model_type
model_param_dict = model_param.model_param_dict
model_class = TrainTestModel.find_subclass(model_type)
train_xys = model_class.get_xys_from_results(train_features)
train_xs = model_class.get_xs_from_results(train_features)
train_ys = model_class.get_ys_from_results(train_features)
model = model_class(model_param_dict, logger)
model.train(train_xys)
# append additional information to model before saving, so that
# VmafQualityRunner can read and process
model.append_info('feature_dict', feature_param.feature_dict)
if 'score_clip' in model_param_dict:
VmafQualityRunner.set_clip_score(model, model_param_dict['score_clip'])
if 'score_transform' in model_param_dict:
VmafQualityRunner.set_transform_score(model, model_param_dict['score_transform'])
train_ys_pred = VmafQualityRunner.predict_with_model(model, train_xs, **kwargs)
raw_groundtruths = None if train_raw_assets is None else \
map(lambda asset: asset.raw_groundtruth, train_raw_assets)
train_stats = model.get_stats(train_ys['label'], train_ys_pred,
ys_label_raw=raw_groundtruths)
log = 'Stats on training data: {}'.format(model.format_stats(train_stats))
if logger:
logger.info(log)
else:
print log
# save model
if output_model_filepath is not None:
model.to_file(output_model_filepath)
if train_ax is not None:
train_content_ids = map(lambda asset: asset.content_id, train_assets)
model_class.plot_scatter(train_ax, train_stats, train_content_ids)
train_ax.set_xlabel('True Score')
train_ax.set_ylabel("Predicted Score")
train_ax.grid()
train_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=train_dataset.dataset_name,
model=model.model_id,
stats=model_class.format_stats(train_stats)
))
# === test model on test dataset ===
if test_dataset is None:
test_assets = None
test_stats = None
test_fassembler = None
else:
test_assets = read_dataset(test_dataset, **kwargs)
test_raw_assets = None
try:
for test_asset in test_assets:
assert test_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = kwargs['subj_model_class'] if 'subj_model_class' in kwargs and kwargs['subj_model_class'] is not None else DmosModel
subjective_model = subj_model_class(RawDatasetReader(test_dataset))
subjective_model.run_modeling(**kwargs)
test_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs)
test_raw_assets = test_assets
test_assets = read_dataset(test_dataset_aggregate, **kwargs)
test_fassembler = FeatureAssembler(
feature_dict = feature_param.feature_dict,
feature_option_dict = None,
assets = test_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_features = test_fassembler.results
for result in test_features:
result.set_score_aggregate_method(aggregate_method)
test_xs = model_class.get_xs_from_results(test_features)
test_ys = model_class.get_ys_from_results(test_features)
test_ys_pred = VmafQualityRunner.predict_with_model(model, test_xs, **kwargs)
raw_groundtruths = None if test_raw_assets is None else \
map(lambda asset: asset.raw_groundtruth, test_raw_assets)
test_stats = model_class.get_stats(test_ys['label'], test_ys_pred,
ys_label_raw=raw_groundtruths)
log = 'Stats on testing data: {}'.format(model_class.format_stats(test_stats))
if logger:
logger.info(log)
else:
print log
if test_ax is not None:
test_content_ids = map(lambda asset: asset.content_id, test_assets)
model_class.plot_scatter(test_ax, test_stats, test_content_ids)
test_ax.set_xlabel('True Score')
test_ax.set_ylabel("Predicted Score")
test_ax.grid()
test_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=test_dataset.dataset_name,
model=model.model_id,
stats=model_class.format_stats(test_stats)
))
return train_fassembler, train_assets, train_stats, \
test_fassembler, test_assets, test_stats, model
def train_test_vmaf_on_dataset(
train_dataset,
test_dataset,
feature_param,
model_param,
train_ax,
test_ax,
result_store,
parallelize=True,
logger=None,
fifo_mode=True,
output_model_filepath=None,
aggregate_method=np.mean,
**kwargs
):
train_assets = read_dataset(train_dataset, **kwargs)
train_raw_assets = None
try:
for train_asset in train_assets:
assert train_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = (
kwargs["subj_model_class"]
if "subj_model_class" in kwargs and kwargs["subj_model_class"] is not None
else DmosModel
)
subjective_model = subj_model_class(RawDatasetReader(train_dataset))
subjective_model.run_modeling(**kwargs)
train_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs)
train_raw_assets = train_assets
train_assets = read_dataset(train_dataset_aggregate, **kwargs)
train_fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=train_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=parallelize,
)
train_fassembler.run()
train_features = train_fassembler.results
for result in train_features:
result.set_score_aggregate_method(aggregate_method)
model_type = model_param.model_type
model_param_dict = model_param.model_param_dict
model_class = TrainTestModel.find_subclass(model_type)
train_xys = model_class.get_xys_from_results(train_features)
train_xs = model_class.get_xs_from_results(train_features)
train_ys = model_class.get_ys_from_results(train_features)
model = model_class(model_param_dict, logger)
model.train(train_xys)
# append additional information to model before saving, so that
# VmafQualityRunner can read and process
model.append_info("feature_dict", feature_param.feature_dict)
if "score_clip" in model_param_dict:
VmafQualityRunner.set_clip_score(model, model_param_dict["score_clip"])
train_ys_pred = VmafQualityRunner.predict_with_model(model, train_xs, **kwargs)
raw_groundtruths = None if train_raw_assets is None else map(lambda asset: asset.raw_groundtruth, train_raw_assets)
train_stats = model.get_stats(train_ys["label"], train_ys_pred, ys_label_raw=raw_groundtruths)
log = "Stats on training data: {}".format(model.format_stats(train_stats))
if logger:
logger.info(log)
else:
print log
# save model
if output_model_filepath is not None:
model.to_file(output_model_filepath)
if train_ax is not None:
train_content_ids = map(lambda asset: asset.content_id, train_assets)
model_class.plot_scatter(train_ax, train_stats, train_content_ids)
train_ax.set_xlabel("True Score")
train_ax.set_ylabel("Predicted Score")
train_ax.grid()
train_ax.set_title(
"Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=train_dataset.dataset_name, model=model.model_id, stats=model_class.format_stats(train_stats)
)
)
# === test model on test dataset ===
if test_dataset is None:
test_assets = None
test_stats = None
test_fassembler = None
else:
test_assets = read_dataset(test_dataset, **kwargs)
test_raw_assets = None
try:
for test_asset in test_assets:
assert test_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
subj_model_class = (
kwargs["subj_model_class"]
if "subj_model_class" in kwargs and kwargs["subj_model_class"] is not None
else DmosModel
)
subjective_model = subj_model_class(RawDatasetReader(test_dataset))
subjective_model.run_modeling(**kwargs)
test_dataset_aggregate = subjective_model.to_aggregated_dataset(**kwargs)
test_raw_assets = test_assets
test_assets = read_dataset(test_dataset_aggregate, **kwargs)
test_fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=test_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
test_fassembler.run()
test_features = test_fassembler.results
for result in test_features:
result.set_score_aggregate_method(aggregate_method)
test_xs = model_class.get_xs_from_results(test_features)
test_ys = model_class.get_ys_from_results(test_features)
test_ys_pred = VmafQualityRunner.predict_with_model(model, test_xs, **kwargs)
raw_groundtruths = (
None if test_raw_assets is None else map(lambda asset: asset.raw_groundtruth, test_raw_assets)
)
test_stats = model_class.get_stats(test_ys["label"], test_ys_pred, ys_label_raw=raw_groundtruths)
log = "Stats on testing data: {}".format(model_class.format_stats(test_stats))
if logger:
logger.info(log)
else:
print log
if test_ax is not None:
test_content_ids = map(lambda asset: asset.content_id, test_assets)
model_class.plot_scatter(test_ax, test_stats, test_content_ids)
test_ax.set_xlabel("True Score")
test_ax.set_ylabel("Predicted Score")
test_ax.grid()
test_ax.set_title(
"Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=test_dataset.dataset_name, model=model.model_id, stats=model_class.format_stats(test_stats)
)
)
return train_fassembler, train_assets, train_stats, test_fassembler, test_assets, test_stats, model
def train_test_on_dataset(train_dataset, test_dataset,
feature_param, model_param,
train_ax, test_ax, result_store,
parallelize=True, logger=None, fifo_mode=True,
output_model_filepath=None):
train_assets = read_dataset(train_dataset)
train_fassembler = FeatureAssembler(
feature_dict = feature_param.feature_dict,
feature_option_dict = None,
assets = train_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
parallelize=parallelize,
)
train_fassembler.run()
train_features = train_fassembler.results
train_xys = TrainTestModel.get_xys_from_results(train_features)
train_xs = TrainTestModel.get_xs_from_results(train_features)
train_ys = TrainTestModel.get_ys_from_results(train_features)
model_type = model_param.model_type
model_param_dict = model_param.model_param_dict
model_class = TrainTestModel.find_subclass(model_type)
model = model_class(model_param_dict, logger)
model.train(train_xys)
# append additional information to model before saving, so that
# VmafQualityRunner can read and process
model.append_info('feature_dict', feature_param.feature_dict)
if 'score_clip' in model_param_dict:
VmafQualityRunner.set_clip_score(model, model_param_dict['score_clip'])
train_ys_pred = model.predict(train_xs)
# apply instructions indicated in the appended info
train_ys_pred = VmafQualityRunner.clip_score(model, train_ys_pred)
train_stats = TrainTestModel.get_stats(train_ys['label'], train_ys_pred)
if logger:
logger.info('Stats on training data: {}'.format(TrainTestModel.
format_stats(train_stats)))
# save model
if output_model_filepath is not None:
model.to_file(output_model_filepath)
if train_ax is not None:
train_content_ids = map(lambda asset: asset.content_id, train_assets)
TrainTestModel.plot_scatter(train_ax, train_stats, train_content_ids)
train_ax.set_xlabel('DMOS')
train_ax.set_ylabel("Predicted Score")
train_ax.grid()
train_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=train_dataset.dataset_name,
model=model.model_id,
stats=TrainTestModel.format_stats(train_stats)
))
# === test model on test dataset ===
if test_dataset is None:
test_assets = None
test_stats = None
test_fassembler = None
else:
test_assets = read_dataset(test_dataset)
test_fassembler = FeatureAssembler(
feature_dict = feature_param.feature_dict,
feature_option_dict = None,
assets = test_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
parallelize=True,
)
test_fassembler.run()
test_features = test_fassembler.results
test_xs = TrainTestModel.get_xs_from_results(test_features)
test_ys = TrainTestModel.get_ys_from_results(test_features)
test_ys_pred = model.predict(test_xs)
# apply instructions indicated in the appended info
test_ys_pred = VmafQualityRunner.clip_score(model, test_ys_pred)
test_stats = TrainTestModel.get_stats(test_ys['label'], test_ys_pred)
if logger:
logger.info('Stats on testing data: {}'.format(
TrainTestModel.format_stats(test_stats)))
if test_ax is not None:
test_content_ids = map(lambda asset: asset.content_id, test_assets)
TrainTestModel.plot_scatter(test_ax, test_stats, test_content_ids)
test_ax.set_xlabel('DMOS')
test_ax.set_ylabel("Predicted Score")
test_ax.grid()
test_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=test_dataset.dataset_name,
model=model.model_id,
stats=TrainTestModel.format_stats(test_stats)
))
return train_fassembler, train_assets, train_stats, \
test_fassembler, test_assets, test_stats
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature': 'all'},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44609306249999997,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9254334398006141,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
23.509571520833333,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
31.271439270833337,
places=4)
class FeatureAssemblerTest(unittest.TestCase):
def tearDown(self):
if hasattr(self, 'fassembler'):
self.fassembler.remove_results()
pass
def test_get_fextractor_subclasses(self):
from core.noref_feature_extractor import NorefFeatureExtractor
fextractor_subclasses = FeatureExtractor.get_subclasses_recursively()
self.assertEquals(len(fextractor_subclasses), 7)
self.assertTrue(VmafFeatureExtractor in fextractor_subclasses)
self.assertTrue(MomentFeatureExtractor in fextractor_subclasses)
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature': 'all'},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44609306249999997,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'],
0.9254334398006141,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'],
23.509571520833333,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'],
31.271439270833337,
places=4)
def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test",
content_id=0,
asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={
'width': 576,
'height': 324
})
asset_original = Asset(dataset="test",
content_id=0,
asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={
'width': 576,
'height': 324
})
self.fassembler = FeatureAssembler(
feature_dict={'VMAF_feature': ['vif', 'motion']},
feature_option_dict=None,
assets=[asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
optional_dict=None,
optional_dict2=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'],
0.44609306249999997,
places=4)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'],
1.0,
places=4)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'],
4.0498253541666669,
places=4)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
def train_test_on_dataset(train_dataset,
test_dataset,
feature_param,
model_param,
train_ax,
test_ax,
result_store,
parallelize=True,
logger=None,
fifo_mode=True,
output_model_filepath=None):
train_assets = read_dataset(train_dataset)
train_fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=train_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
parallelize=parallelize,
)
train_fassembler.run()
train_features = train_fassembler.results
train_xys = TrainTestModel.get_xys_from_results(train_features)
train_xs = TrainTestModel.get_xs_from_results(train_features)
train_ys = TrainTestModel.get_ys_from_results(train_features)
model_type = model_param.model_type
model_param_dict = model_param.model_param_dict
model_class = TrainTestModel.find_subclass(model_type)
model = model_class(model_param_dict, logger)
model.train(train_xys)
# append additional information to model before saving, so that
# VmafQualityRunner can read and process
model.append_info('feature_dict', feature_param.feature_dict)
if 'score_clip' in model_param_dict:
VmafQualityRunner.set_clip_score(model, model_param_dict['score_clip'])
train_ys_pred = model.predict(train_xs)
# apply instructions indicated in the appended info
train_ys_pred = VmafQualityRunner.clip_score(model, train_ys_pred)
train_stats = TrainTestModel.get_stats(train_ys['label'], train_ys_pred)
if logger:
logger.info('Stats on training data: {}'.format(
TrainTestModel.format_stats(train_stats)))
# save model
if output_model_filepath is not None:
model.to_file(output_model_filepath)
if train_ax is not None:
train_content_ids = map(lambda asset: asset.content_id, train_assets)
TrainTestModel.plot_scatter(train_ax, train_stats, train_content_ids)
train_ax.set_xlabel('DMOS')
train_ax.set_ylabel("Predicted Score")
train_ax.grid()
train_ax.set_title(
"Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=train_dataset.dataset_name,
model=model.model_id,
stats=TrainTestModel.format_stats(train_stats)))
# === test model on test dataset ===
if test_dataset is None:
test_assets = None
test_stats = None
test_fassembler = None
else:
test_assets = read_dataset(test_dataset)
test_fassembler = FeatureAssembler(
feature_dict=feature_param.feature_dict,
feature_option_dict=None,
assets=test_assets,
logger=logger,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
parallelize=True,
)
test_fassembler.run()
test_features = test_fassembler.results
test_xs = TrainTestModel.get_xs_from_results(test_features)
test_ys = TrainTestModel.get_ys_from_results(test_features)
test_ys_pred = model.predict(test_xs)
# apply instructions indicated in the appended info
test_ys_pred = VmafQualityRunner.clip_score(model, test_ys_pred)
test_stats = TrainTestModel.get_stats(test_ys['label'], test_ys_pred)
if logger:
logger.info('Stats on testing data: {}'.format(
TrainTestModel.format_stats(test_stats)))
if test_ax is not None:
test_content_ids = map(lambda asset: asset.content_id, test_assets)
TrainTestModel.plot_scatter(test_ax, test_stats, test_content_ids)
test_ax.set_xlabel('DMOS')
test_ax.set_ylabel("Predicted Score")
test_ax.grid()
test_ax.set_title(
"Dataset: {dataset}, Model: {model}\n{stats}".format(
dataset=test_dataset.dataset_name,
model=model.model_id,
stats=TrainTestModel.format_stats(test_stats)))
return train_fassembler, train_assets, train_stats, \
test_fassembler, test_assets, test_stats
class FeatureAssemblerTest(unittest.TestCase):
def tearDown(self):
if hasattr(self, 'fassembler'):
self.fassembler.remove_results()
pass
def test_get_fextractor_subclasses(self):
fextractor_subclasses = FeatureExtractor.get_subclasses_recursively()
self.assertEquals(len(fextractor_subclasses), 3)
self.assertTrue(VmafFeatureExtractor in fextractor_subclasses)
self.assertTrue(MomentFeatureExtractor in fextractor_subclasses)
def test_feature_assembler_whole_feature(self):
print 'test on feature assembler with whole feature...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict = {'VMAF_feature':'all'},
feature_option_dict = None,
assets = [asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[0]['VMAF_feature_adm2_score'], 0.9254334398006141)
self.assertAlmostEqual(results[0]['VMAF_feature_ansnr_score'], 22.533456770833329)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_adm2_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_ansnr_score'], 30.030914145833322)
def test_feature_assembler_selected_atom_feature(self):
print 'test on feature assembler with selected atom features...'
ref_path = config.ROOT + "/resource/yuv/src01_hrc00_576x324.yuv"
dis_path = config.ROOT + "/resource/yuv/src01_hrc01_576x324.yuv"
asset = Asset(dataset="test", content_id=0, asset_id=0,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=dis_path,
asset_dict={'width':576, 'height':324})
asset_original = Asset(dataset="test", content_id=0, asset_id=1,
workdir_root=config.ROOT + "/workspace/workdir",
ref_path=ref_path,
dis_path=ref_path,
asset_dict={'width':576, 'height':324})
self.fassembler = FeatureAssembler(
feature_dict = {'VMAF_feature':['vif', 'motion']},
feature_option_dict = None,
assets = [asset, asset_original],
logger=None,
fifo_mode=True,
delete_workdir=True,
result_store=None,
parallelize=True,
)
self.fassembler.run()
results = self.fassembler.results
self.assertAlmostEqual(results[0]['VMAF_feature_vif_score'], 0.44455808333333313)
self.assertAlmostEqual(results[0]['VMAF_feature_motion_score'], 3.5916076041666667)
self.assertAlmostEqual(results[1]['VMAF_feature_vif_score'], 1.0)
self.assertAlmostEqual(results[1]['VMAF_feature_motion_score'], 3.5916076041666667)
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_ansnr_score']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_scores']
with self.assertRaises(KeyError):
results[0]['VMAF_feature_adm_score']
