def to_dataframe(self):
"""
Export to pandas dataframe with columns: dataset, content_id, asset_id,
ref_name, dis_name, asset, executor_id, scores_key, scores
Example:
asset asset_id content_id \
0 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
1 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
2 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
3 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
4 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
dataset dis_name executor_id \
0 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
1 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
2 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
3 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
4 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
ref_name \
0 checkerboard_1920_1080_10_3_0_0.yuv
1 checkerboard_1920_1080_10_3_0_0.yuv
2 checkerboard_1920_1080_10_3_0_0.yuv
3 checkerboard_1920_1080_10_3_0_0.yuv
4 checkerboard_1920_1080_10_3_0_0.yuv
scores scores_key
0 [0.798588, 0.84287, 0.800122] VMAF_adm_scores
1 [12.420815, 12.41775, 12.416308] VMAF_ansnr_scores
2 [0.0, 18.489031, 18.542355] VMAF_motion_scores
3 [42.1117149479, 47.6544689539, 40.6168118533] VMAF_scores
4 [0.156106, 0.156163, 0.156119] VMAF_vif_scores
[5 rows x 9 columns]
:return:
"""
import pandas as pd
asset = self.asset
executor_id = self.executor_id
list_scores_key = self.get_ordered_list_scores_key()
list_scores = list(map(lambda key: self.result_dict[key], list_scores_key))
rows = []
for scores_key, scores in zip(list_scores_key, list_scores):
row = [asset.dataset,
asset.content_id,
asset.asset_id,
get_file_name_with_extension(asset.ref_path),
get_file_name_with_extension(asset.dis_path),
repr(asset),
executor_id,
scores_key,
scores]
rows.append(row)
# zip rows into a dict, and wrap with df
df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows))))
return df
def to_dataframe(self):
"""
Export to pandas dataframe with columns: dataset, content_id, asset_id,
ref_name, dis_name, asset, executor_id, scores_key, scores
Example:
asset asset_id content_id \
0 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
1 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
2 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
3 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
4 {"asset_dict": {"height": 1080, "width": 1920}... 0 0
dataset dis_name executor_id \
0 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
1 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
2 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
3 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
4 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1
ref_name \
0 checkerboard_1920_1080_10_3_0_0.yuv
1 checkerboard_1920_1080_10_3_0_0.yuv
2 checkerboard_1920_1080_10_3_0_0.yuv
3 checkerboard_1920_1080_10_3_0_0.yuv
4 checkerboard_1920_1080_10_3_0_0.yuv
scores scores_key
0 [0.798588, 0.84287, 0.800122] VMAF_adm_scores
1 [12.420815, 12.41775, 12.416308] VMAF_ansnr_scores
2 [0.0, 18.489031, 18.542355] VMAF_motion_scores
3 [42.1117149479, 47.6544689539, 40.6168118533] VMAF_scores
4 [0.156106, 0.156163, 0.156119] VMAF_vif_scores
[5 rows x 9 columns]
:return:
"""
import pandas as pd
asset = self.asset
executor_id = self.executor_id
list_scores_key = self.get_ordered_list_scores_key()
list_scores = map(lambda key: self.result_dict[key], list_scores_key)
rows = []
for scores_key, scores in zip(list_scores_key, list_scores):
row = [asset.dataset,
asset.content_id,
asset.asset_id,
get_file_name_with_extension(asset.ref_path),
get_file_name_with_extension(asset.dis_path),
repr(asset),
executor_id,
scores_key,
scores]
rows.append(row)
# zip rows into a dict, and wrap with df
df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows))))
return df
def to_dataframe(self):
import pandas as pd
asset = self.asset
executor_id = self.executor_id
list_scores_key = self.get_ordered_list_scores_key()
list_scores = list(
map(lambda key: self.result_dict[key], list_scores_key))
rows = []
for scores_key, scores in zip(list_scores_key, list_scores):
row = [
asset.dataset, asset.content_id, asset.asset_id,
get_file_name_with_extension(asset.ref_path),
get_file_name_with_extension(asset.dis_path),
repr(asset), executor_id, scores_key, scores
]
rows.append(row)
# zip rows into a dict, and wrap with df
df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows))))
return df
def to_normalized_dict(self):
"""
Similar to self.__dict__ except for excluding workdir (which is random)
and dir part of ref_path/dis_path.
:return:
"""
d = {}
for key in self.__dict__:
if key == 'workdir':
d[key] = ""
elif key == 'ref_path' or key == 'dis_path':
d[key] = get_file_name_with_extension(self.__dict__[key])
else:
d[key] = self.__dict__[key]
return d
def main():
if len(sys.argv) < 3:
print_usage()
return 2
try:
subjective_model = sys.argv[1]
dataset_filepath = sys.argv[2]
except ValueError:
print_usage()
return 2
try:
subjective_model_class = SubjectiveModel.find_subclass(
subjective_model)
except Exception as e:
print "Error: " + str(e)
return 1
print "Run model {} on dataset {}".format(
subjective_model_class.__name__,
get_file_name_with_extension(dataset_filepath))
run_subjective_models(
dataset_filepath=dataset_filepath,
subjective_model_classes=[
subjective_model_class,
],
normalize_final=False, # True or False
do_plot=[
'raw_scores',
'quality_scores',
'subject_scores',
'content_scores',
],
plot_type='errorbar',
gradient_method='simplified',
)
plt.show()
return 0
def run_test_on_dataset(test_dataset,
runner_class,
ax,
result_store,
model_filepath,
parallelize=True,
fifo_mode=True,
aggregate_method=np.mean,
type='regressor',
**kwargs):
test_assets = read_dataset(test_dataset, **kwargs)
test_raw_assets = None
try:
for test_asset in test_assets:
assert test_asset.groundtruth is not None
except AssertionError:
# no groundtruth, try do subjective modeling
from sureal.dataset_reader import RawDatasetReader
from sureal.subjective_model import DmosModel
subj_model_class = kwargs[
'subj_model_class'] if 'subj_model_class' in kwargs and kwargs[
'subj_model_class'] is not None else DmosModel
dataset_reader_class = kwargs[
'dataset_reader_class'] if 'dataset_reader_class' in kwargs else RawDatasetReader
subjective_model = subj_model_class(dataset_reader_class(test_dataset))
subjective_model.run_modeling(**kwargs)
test_dataset_aggregate = subjective_model.to_aggregated_dataset(
**kwargs)
test_raw_assets = test_assets
test_assets = read_dataset(test_dataset_aggregate, **kwargs)
if model_filepath is not None:
optional_dict = {'model_filepath': model_filepath}
if 'model_720_filepath' in kwargs and kwargs[
'model_720_filepath'] is not None:
optional_dict['720model_filepath'] = kwargs['model_720_filepath']
if 'model_480_filepath' in kwargs and kwargs[
'model_480_filepath'] is not None:
optional_dict['480model_filepath'] = kwargs['model_480_filepath']
else:
optional_dict = None
if 'enable_transform_score' in kwargs and kwargs[
'enable_transform_score'] is not None:
if not optional_dict:
optional_dict = {}
optional_dict['enable_transform_score'] = kwargs[
'enable_transform_score']
if 'disable_clip_score' in kwargs and kwargs[
'disable_clip_score'] is not None:
if not optional_dict:
optional_dict = {}
optional_dict['disable_clip_score'] = kwargs['disable_clip_score']
if 'subsample' in kwargs and kwargs['subsample'] is not None:
if not optional_dict:
optional_dict = {}
optional_dict['subsample'] = kwargs['subsample']
if 'additional_optional_dict' in kwargs and kwargs[
'additional_optional_dict'] is not None:
assert isinstance(kwargs['additional_optional_dict'], dict)
if not optional_dict:
optional_dict = {}
optional_dict.update(kwargs['additional_optional_dict'])
if 'processes' in kwargs and kwargs['processes'] is not None:
assert isinstance(kwargs['processes'], int)
processes = kwargs['processes']
else:
processes = None
if processes is not None:
assert parallelize is True
# run
runner = runner_class(
test_assets,
None,
fifo_mode=fifo_mode,
delete_workdir=True,
result_store=result_store,
optional_dict=optional_dict,
optional_dict2=None,
)
runner.run(parallelize=parallelize, processes=processes)
results = runner.results
list_score_key = results[0].get_ordered_list_score_key()
COLUMNS = [
'dataset', 'content_id', 'asset_id', 'ref_name', 'dis_name', 'width',
'height', 'DMOS'
]
COLUMNS.extend(list_score_key)
df = pd.DataFrame(columns=tuple(COLUMNS))
for result in results:
result.set_score_aggregate_method(aggregate_method)
rows = []
# result[runner_class.get_score_key()]
list_score_key = result.get_ordered_list_score_key()
list_aggregate_score = list(
map(lambda key: result[key], list_score_key))
total_score = list_aggregate_score[-1]
#width,height=result.asset.quality_width_height()
#for score_key, score in zip(list_score_key[:-1], list_aggregate_score[:-1]):
row = [
result.asset.dataset, result.asset.content_id,
result.asset.asset_id,
get_file_name_with_extension(result.asset.ref_path),
get_file_name_with_extension(result.asset.dis_path),
result.asset.asset_dict['dis_width'],
result.asset.asset_dict['dis_height'], result.asset.groundtruth
]
row.extend(list_aggregate_score)
#rows.append(row)
df2 = pd.DataFrame([row], columns=COLUMNS)
df = df.append(df2)
df.to_csv('/Users/jessica/CMT309/Project-VMAF/vmaf-master/result_4k.csv')
try:
model_type = runner.get_train_test_model_class()
except:
if type == 'regressor':
model_type = RegressorMixin
elif type == 'classifier':
model_type = ClassifierMixin
else:
assert False
split_test_indices_for_perf_ci = kwargs['split_test_indices_for_perf_ci'] \
if 'split_test_indices_for_perf_ci' in kwargs else False
# plot
groundtruths = list(map(lambda asset: asset.groundtruth, test_assets))
predictions = list(
map(lambda result: result[runner_class.get_score_key()], results))
raw_grountruths = None if test_raw_assets is None else \
list(map(lambda asset: asset.raw_groundtruth, test_raw_assets))
groundtruths_std = None if test_assets is None else \
list(map(lambda asset: asset.groundtruth_std, test_assets))
try:
predictions_bagging = list(
map(lambda result: result[runner_class.get_bagging_score_key()],
results))
predictions_stddev = list(
map(lambda result: result[runner_class.get_stddev_score_key()],
results))
predictions_ci95_low = list(
map(lambda result: result[runner_class.get_ci95_low_score_key()],
results))
predictions_ci95_high = list(
map(lambda result: result[runner_class.get_ci95_high_score_key()],
results))
predictions_all_models = list(
map(lambda result: result[runner_class.get_all_models_score_key()],
results))
# need to revert the list of lists, so that the outer list has the predictions for each model separately
predictions_all_models = np.array(predictions_all_models).T.tolist()
num_models = np.shape(predictions_all_models)[0]
stats = model_type.get_stats(
groundtruths,
predictions,
ys_label_raw=raw_grountruths,
ys_label_pred_bagging=predictions_bagging,
ys_label_pred_stddev=predictions_stddev,
ys_label_pred_ci95_low=predictions_ci95_low,
ys_label_pred_ci95_high=predictions_ci95_high,
ys_label_pred_all_models=predictions_all_models,
ys_label_stddev=groundtruths_std,
split_test_indices_for_perf_ci=split_test_indices_for_perf_ci)
except Exception as e:
print(
'Warning: stats calculation failed, fall back to default stats calculation: {}'
.format(e))
stats = model_type.get_stats(
groundtruths,
predictions,
ys_label_raw=raw_grountruths,
ys_label_stddev=groundtruths_std,
split_test_indices_for_perf_ci=split_test_indices_for_perf_ci)
num_models = 1
print('Stats on testing data: {}'.format(
model_type.format_stats_for_print(stats)))
# printing stats if multiple models are present
if 'SRCC_across_model_distribution' in stats \
and 'PCC_across_model_distribution' in stats \
and 'RMSE_across_model_distribution' in stats:
print(
'Stats on testing data (across multiple models, using all test indices): {}'
.format(
model_type.format_across_model_stats_for_print(
model_type.extract_across_model_stats(stats))))
if split_test_indices_for_perf_ci:
print('Stats on testing data (single model, multiple test sets): {}'.
format(
model_type.format_stats_across_test_splits_for_print(
model_type.extract_across_test_splits_stats(stats))))
if ax is not None:
content_ids = list(map(lambda asset: asset.content_id, test_assets))
if 'point_label' in kwargs:
if kwargs['point_label'] == 'asset_id':
point_labels = list(
map(lambda asset: asset.asset_id, test_assets))
elif kwargs['point_label'] == 'dis_path':
point_labels = list(
map(
lambda asset: get_file_name_without_extension(
asset.dis_path), test_assets))
else:
raise AssertionError("Unknown point_label {}".format(
kwargs['point_label']))
else:
point_labels = None
model_type.plot_scatter(ax,
stats,
content_ids=content_ids,
point_labels=point_labels,
**kwargs)
ax.set_xlabel('True Score')
ax.set_ylabel("Predicted Score")
ax.grid()
ax.set_title("{runner}{num_models}\n{stats}".format(
dataset=test_assets[0].dataset,
runner=runner_class.TYPE,
stats=model_type.format_stats_for_plot(stats),
num_models=", {} models".format(num_models)
if num_models > 1 else "",
))
return test_assets, results
