def to_dataframe(self): """ Export to pandas dataframe with columns: dataset, content_id, asset_id, ref_name, dis_name, asset, executor_id, scores_key, scores Example: asset asset_id content_id \ 0 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 1 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 2 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 3 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 4 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 dataset dis_name executor_id \ 0 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 1 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 2 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 3 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 4 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 ref_name \ 0 checkerboard_1920_1080_10_3_0_0.yuv 1 checkerboard_1920_1080_10_3_0_0.yuv 2 checkerboard_1920_1080_10_3_0_0.yuv 3 checkerboard_1920_1080_10_3_0_0.yuv 4 checkerboard_1920_1080_10_3_0_0.yuv scores scores_key 0 [0.798588, 0.84287, 0.800122] VMAF_adm_scores 1 [12.420815, 12.41775, 12.416308] VMAF_ansnr_scores 2 [0.0, 18.489031, 18.542355] VMAF_motion_scores 3 [42.1117149479, 47.6544689539, 40.6168118533] VMAF_scores 4 [0.156106, 0.156163, 0.156119] VMAF_vif_scores [5 rows x 9 columns] :return: """ import pandas as pd asset = self.asset executor_id = self.executor_id list_scores_key = self.get_ordered_list_scores_key() list_scores = list(map(lambda key: self.result_dict[key], list_scores_key)) rows = [] for scores_key, scores in zip(list_scores_key, list_scores): row = [asset.dataset, asset.content_id, asset.asset_id, get_file_name_with_extension(asset.ref_path), get_file_name_with_extension(asset.dis_path), repr(asset), executor_id, scores_key, scores] rows.append(row) # zip rows into a dict, and wrap with df df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows)))) return df
def to_dataframe(self): """ Export to pandas dataframe with columns: dataset, content_id, asset_id, ref_name, dis_name, asset, executor_id, scores_key, scores Example: asset asset_id content_id \ 0 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 1 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 2 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 3 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 4 {"asset_dict": {"height": 1080, "width": 1920}... 0 0 dataset dis_name executor_id \ 0 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 1 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 2 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 3 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 4 test checkerboard_1920_1080_10_3_1_0.yuv VMAF_V0.1 ref_name \ 0 checkerboard_1920_1080_10_3_0_0.yuv 1 checkerboard_1920_1080_10_3_0_0.yuv 2 checkerboard_1920_1080_10_3_0_0.yuv 3 checkerboard_1920_1080_10_3_0_0.yuv 4 checkerboard_1920_1080_10_3_0_0.yuv scores scores_key 0 [0.798588, 0.84287, 0.800122] VMAF_adm_scores 1 [12.420815, 12.41775, 12.416308] VMAF_ansnr_scores 2 [0.0, 18.489031, 18.542355] VMAF_motion_scores 3 [42.1117149479, 47.6544689539, 40.6168118533] VMAF_scores 4 [0.156106, 0.156163, 0.156119] VMAF_vif_scores [5 rows x 9 columns] :return: """ import pandas as pd asset = self.asset executor_id = self.executor_id list_scores_key = self.get_ordered_list_scores_key() list_scores = map(lambda key: self.result_dict[key], list_scores_key) rows = [] for scores_key, scores in zip(list_scores_key, list_scores): row = [asset.dataset, asset.content_id, asset.asset_id, get_file_name_with_extension(asset.ref_path), get_file_name_with_extension(asset.dis_path), repr(asset), executor_id, scores_key, scores] rows.append(row) # zip rows into a dict, and wrap with df df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows)))) return df
def to_dataframe(self): import pandas as pd asset = self.asset executor_id = self.executor_id list_scores_key = self.get_ordered_list_scores_key() list_scores = list( map(lambda key: self.result_dict[key], list_scores_key)) rows = [] for scores_key, scores in zip(list_scores_key, list_scores): row = [ asset.dataset, asset.content_id, asset.asset_id, get_file_name_with_extension(asset.ref_path), get_file_name_with_extension(asset.dis_path), repr(asset), executor_id, scores_key, scores ] rows.append(row) # zip rows into a dict, and wrap with df df = pd.DataFrame(dict(zip(self.DATAFRAME_COLUMNS, zip(*rows)))) return df
def to_normalized_dict(self): """ Similar to self.__dict__ except for excluding workdir (which is random) and dir part of ref_path/dis_path. :return: """ d = {} for key in self.__dict__: if key == 'workdir': d[key] = "" elif key == 'ref_path' or key == 'dis_path': d[key] = get_file_name_with_extension(self.__dict__[key]) else: d[key] = self.__dict__[key] return d
def main(): if len(sys.argv) < 3: print_usage() return 2 try: subjective_model = sys.argv[1] dataset_filepath = sys.argv[2] except ValueError: print_usage() return 2 try: subjective_model_class = SubjectiveModel.find_subclass( subjective_model) except Exception as e: print "Error: " + str(e) return 1 print "Run model {} on dataset {}".format( subjective_model_class.__name__, get_file_name_with_extension(dataset_filepath)) run_subjective_models( dataset_filepath=dataset_filepath, subjective_model_classes=[ subjective_model_class, ], normalize_final=False, # True or False do_plot=[ 'raw_scores', 'quality_scores', 'subject_scores', 'content_scores', ], plot_type='errorbar', gradient_method='simplified', ) plt.show() return 0
def run_test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean, type='regressor', **kwargs): test_assets = read_dataset(test_dataset, **kwargs) test_raw_assets = None try: for test_asset in test_assets: assert test_asset.groundtruth is not None except AssertionError: # no groundtruth, try do subjective modeling from sureal.dataset_reader import RawDatasetReader from sureal.subjective_model import DmosModel subj_model_class = kwargs[ 'subj_model_class'] if 'subj_model_class' in kwargs and kwargs[ 'subj_model_class'] is not None else DmosModel dataset_reader_class = kwargs[ 'dataset_reader_class'] if 'dataset_reader_class' in kwargs else RawDatasetReader subjective_model = subj_model_class(dataset_reader_class(test_dataset)) subjective_model.run_modeling(**kwargs) test_dataset_aggregate = subjective_model.to_aggregated_dataset( **kwargs) test_raw_assets = test_assets test_assets = read_dataset(test_dataset_aggregate, **kwargs) if model_filepath is not None: optional_dict = {'model_filepath': model_filepath} if 'model_720_filepath' in kwargs and kwargs[ 'model_720_filepath'] is not None: optional_dict['720model_filepath'] = kwargs['model_720_filepath'] if 'model_480_filepath' in kwargs and kwargs[ 'model_480_filepath'] is not None: optional_dict['480model_filepath'] = kwargs['model_480_filepath'] else: optional_dict = None if 'enable_transform_score' in kwargs and kwargs[ 'enable_transform_score'] is not None: if not optional_dict: optional_dict = {} optional_dict['enable_transform_score'] = kwargs[ 'enable_transform_score'] if 'disable_clip_score' in kwargs and kwargs[ 'disable_clip_score'] is not None: if not optional_dict: optional_dict = {} optional_dict['disable_clip_score'] = kwargs['disable_clip_score'] if 'subsample' in kwargs and kwargs['subsample'] is not None: if not optional_dict: optional_dict = {} optional_dict['subsample'] = kwargs['subsample'] if 'additional_optional_dict' in kwargs and kwargs[ 'additional_optional_dict'] is not None: assert isinstance(kwargs['additional_optional_dict'], dict) if not optional_dict: optional_dict = {} optional_dict.update(kwargs['additional_optional_dict']) if 'processes' in kwargs and kwargs['processes'] is not None: assert isinstance(kwargs['processes'], int) processes = kwargs['processes'] else: processes = None if processes is not None: assert parallelize is True # run runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, optional_dict2=None, ) runner.run(parallelize=parallelize, processes=processes) results = runner.results list_score_key = results[0].get_ordered_list_score_key() COLUMNS = [ 'dataset', 'content_id', 'asset_id', 'ref_name', 'dis_name', 'width', 'height', 'DMOS' ] COLUMNS.extend(list_score_key) df = pd.DataFrame(columns=tuple(COLUMNS)) for result in results: result.set_score_aggregate_method(aggregate_method) rows = [] # result[runner_class.get_score_key()] list_score_key = result.get_ordered_list_score_key() list_aggregate_score = list( map(lambda key: result[key], list_score_key)) total_score = list_aggregate_score[-1] #width,height=result.asset.quality_width_height() #for score_key, score in zip(list_score_key[:-1], list_aggregate_score[:-1]): row = [ result.asset.dataset, result.asset.content_id, result.asset.asset_id, get_file_name_with_extension(result.asset.ref_path), get_file_name_with_extension(result.asset.dis_path), result.asset.asset_dict['dis_width'], result.asset.asset_dict['dis_height'], result.asset.groundtruth ] row.extend(list_aggregate_score) #rows.append(row) df2 = pd.DataFrame([row], columns=COLUMNS) df = df.append(df2) df.to_csv('/Users/jessica/CMT309/Project-VMAF/vmaf-master/result_4k.csv') try: model_type = runner.get_train_test_model_class() except: if type == 'regressor': model_type = RegressorMixin elif type == 'classifier': model_type = ClassifierMixin else: assert False split_test_indices_for_perf_ci = kwargs['split_test_indices_for_perf_ci'] \ if 'split_test_indices_for_perf_ci' in kwargs else False # plot groundtruths = list(map(lambda asset: asset.groundtruth, test_assets)) predictions = list( map(lambda result: result[runner_class.get_score_key()], results)) raw_grountruths = None if test_raw_assets is None else \ list(map(lambda asset: asset.raw_groundtruth, test_raw_assets)) groundtruths_std = None if test_assets is None else \ list(map(lambda asset: asset.groundtruth_std, test_assets)) try: predictions_bagging = list( map(lambda result: result[runner_class.get_bagging_score_key()], results)) predictions_stddev = list( map(lambda result: result[runner_class.get_stddev_score_key()], results)) predictions_ci95_low = list( map(lambda result: result[runner_class.get_ci95_low_score_key()], results)) predictions_ci95_high = list( map(lambda result: result[runner_class.get_ci95_high_score_key()], results)) predictions_all_models = list( map(lambda result: result[runner_class.get_all_models_score_key()], results)) # need to revert the list of lists, so that the outer list has the predictions for each model separately predictions_all_models = np.array(predictions_all_models).T.tolist() num_models = np.shape(predictions_all_models)[0] stats = model_type.get_stats( groundtruths, predictions, ys_label_raw=raw_grountruths, ys_label_pred_bagging=predictions_bagging, ys_label_pred_stddev=predictions_stddev, ys_label_pred_ci95_low=predictions_ci95_low, ys_label_pred_ci95_high=predictions_ci95_high, ys_label_pred_all_models=predictions_all_models, ys_label_stddev=groundtruths_std, split_test_indices_for_perf_ci=split_test_indices_for_perf_ci) except Exception as e: print( 'Warning: stats calculation failed, fall back to default stats calculation: {}' .format(e)) stats = model_type.get_stats( groundtruths, predictions, ys_label_raw=raw_grountruths, ys_label_stddev=groundtruths_std, split_test_indices_for_perf_ci=split_test_indices_for_perf_ci) num_models = 1 print('Stats on testing data: {}'.format( model_type.format_stats_for_print(stats))) # printing stats if multiple models are present if 'SRCC_across_model_distribution' in stats \ and 'PCC_across_model_distribution' in stats \ and 'RMSE_across_model_distribution' in stats: print( 'Stats on testing data (across multiple models, using all test indices): {}' .format( model_type.format_across_model_stats_for_print( model_type.extract_across_model_stats(stats)))) if split_test_indices_for_perf_ci: print('Stats on testing data (single model, multiple test sets): {}'. format( model_type.format_stats_across_test_splits_for_print( model_type.extract_across_test_splits_stats(stats)))) if ax is not None: content_ids = list(map(lambda asset: asset.content_id, test_assets)) if 'point_label' in kwargs: if kwargs['point_label'] == 'asset_id': point_labels = list( map(lambda asset: asset.asset_id, test_assets)) elif kwargs['point_label'] == 'dis_path': point_labels = list( map( lambda asset: get_file_name_without_extension( asset.dis_path), test_assets)) else: raise AssertionError("Unknown point_label {}".format( kwargs['point_label'])) else: point_labels = None model_type.plot_scatter(ax, stats, content_ids=content_ids, point_labels=point_labels, **kwargs) ax.set_xlabel('True Score') ax.set_ylabel("Predicted Score") ax.grid() ax.set_title("{runner}{num_models}\n{stats}".format( dataset=test_assets[0].dataset, runner=runner_class.TYPE, stats=model_type.format_stats_for_plot(stats), num_models=", {} models".format(num_models) if num_models > 1 else "", )) return test_assets, results