def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean): test_assets = read_dataset(test_dataset) optional_dict = { 'model_filepath':model_filepath } # construct an quality runner object to assert assets only runner = runner_class(test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, ) try: # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, ) for result in results: result.set_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) stats = TrainTestModel.get_stats(groundtruths, predictions) print 'Stats on testing data: {}'.format(TrainTestModel.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(ax, stats, content_ids) ax.set_xlabel('DMOS') ax.set_ylabel("Predicted Score") ax.grid() # ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format( ax.set_title( "{runner}\n{stats}".format( dataset=test_assets[0].dataset, # runner=results[0].executor_id, runner=runner_class.TYPE, stats=TrainTestModel.format_stats(stats), # stats="", # stats=TrainTestModel.format_stats3(stats), )) except Exception as e: print "Error: " + str(e) return test_assets, results
def test_on_dataset(test_dataset, runner_class, ax, result_store, model_filepath, parallelize=True, fifo_mode=True, aggregate_method=np.mean): test_assets = read_dataset(test_dataset) optional_dict = {'model_filepath': model_filepath} # construct an quality runner object to assert assets only runner = runner_class( test_assets, None, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, optional_dict=optional_dict, ) try: # run _, results = run_executors_in_parallel( runner_class, test_assets, fifo_mode=fifo_mode, delete_workdir=True, parallelize=parallelize, result_store=result_store, optional_dict=optional_dict, ) for result in results: result.set_aggregate_method(aggregate_method) # plot groundtruths = map(lambda asset: asset.groundtruth, test_assets) predictions = map(lambda result: result[runner_class.get_score_key()], results) stats = TrainTestModel.get_stats(groundtruths, predictions) print 'Stats on testing data: {}'.format( TrainTestModel.format_stats(stats)) if ax is not None: content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(ax, stats, content_ids) ax.set_xlabel('DMOS') ax.set_ylabel("Predicted Score") ax.grid() # ax.set_title( "Dataset: {dataset}, Runner: {runner}\n{stats}".format( ax.set_title("{runner}\n{stats}".format( dataset=test_assets[0].dataset, # runner=results[0].executor_id, runner=runner_class.TYPE, stats=TrainTestModel.format_stats(stats), # stats="", # stats=TrainTestModel.format_stats3(stats), )) except Exception as e: print "Error: " + str(e) return test_assets, results
def train_test_on_dataset(train_dataset, test_dataset, feature_param, model_param, train_ax, test_ax, result_store, parallelize=True, logger=None, fifo_mode=True, output_model_filepath=None): train_assets = read_dataset(train_dataset) train_fassembler = FeatureAssembler( feature_dict = feature_param.feature_dict, feature_option_dict = None, assets = train_assets, logger=logger, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, parallelize=parallelize, ) train_fassembler.run() train_features = train_fassembler.results train_xys = TrainTestModel.get_xys_from_results(train_features) train_xs = TrainTestModel.get_xs_from_results(train_features) train_ys = TrainTestModel.get_ys_from_results(train_features) model_type = model_param.model_type model_param_dict = model_param.model_param_dict model_class = TrainTestModel.find_subclass(model_type) model = model_class(model_param_dict, logger) model.train(train_xys) # append additional information to model before saving, so that # VmafQualityRunner can read and process model.append_info('feature_dict', feature_param.feature_dict) if 'score_clip' in model_param_dict: VmafQualityRunner.set_clip_score(model, model_param_dict['score_clip']) train_ys_pred = model.predict(train_xs) # apply instructions indicated in the appended info train_ys_pred = VmafQualityRunner.clip_score(model, train_ys_pred) train_stats = TrainTestModel.get_stats(train_ys['label'], train_ys_pred) if logger: logger.info('Stats on training data: {}'.format(TrainTestModel. format_stats(train_stats))) # save model if output_model_filepath is not None: model.to_file(output_model_filepath) if train_ax is not None: train_content_ids = map(lambda asset: asset.content_id, train_assets) TrainTestModel.plot_scatter(train_ax, train_stats, train_content_ids) train_ax.set_xlabel('DMOS') train_ax.set_ylabel("Predicted Score") train_ax.grid() train_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format( dataset=train_dataset.dataset_name, model=model.model_id, stats=TrainTestModel.format_stats(train_stats) )) # === test model on test dataset === if test_dataset is None: test_assets = None test_stats = None test_fassembler = None else: test_assets = read_dataset(test_dataset) test_fassembler = FeatureAssembler( feature_dict = feature_param.feature_dict, feature_option_dict = None, assets = test_assets, logger=logger, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, parallelize=True, ) test_fassembler.run() test_features = test_fassembler.results test_xs = TrainTestModel.get_xs_from_results(test_features) test_ys = TrainTestModel.get_ys_from_results(test_features) test_ys_pred = model.predict(test_xs) # apply instructions indicated in the appended info test_ys_pred = VmafQualityRunner.clip_score(model, test_ys_pred) test_stats = TrainTestModel.get_stats(test_ys['label'], test_ys_pred) if logger: logger.info('Stats on testing data: {}'.format( TrainTestModel.format_stats(test_stats))) if test_ax is not None: test_content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(test_ax, test_stats, test_content_ids) test_ax.set_xlabel('DMOS') test_ax.set_ylabel("Predicted Score") test_ax.grid() test_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format( dataset=test_dataset.dataset_name, model=model.model_id, stats=TrainTestModel.format_stats(test_stats) )) return train_fassembler, train_assets, train_stats, \ test_fassembler, test_assets, test_stats
def train_test_on_dataset(train_dataset, test_dataset, feature_param, model_param, train_ax, test_ax, result_store, parallelize=True, logger=None, fifo_mode=True, output_model_filepath=None): train_assets = read_dataset(train_dataset) train_fassembler = FeatureAssembler( feature_dict=feature_param.feature_dict, feature_option_dict=None, assets=train_assets, logger=logger, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, parallelize=parallelize, ) train_fassembler.run() train_features = train_fassembler.results train_xys = TrainTestModel.get_xys_from_results(train_features) train_xs = TrainTestModel.get_xs_from_results(train_features) train_ys = TrainTestModel.get_ys_from_results(train_features) model_type = model_param.model_type model_param_dict = model_param.model_param_dict model_class = TrainTestModel.find_subclass(model_type) model = model_class(model_param_dict, logger) model.train(train_xys) # append additional information to model before saving, so that # VmafQualityRunner can read and process model.append_info('feature_dict', feature_param.feature_dict) if 'score_clip' in model_param_dict: VmafQualityRunner.set_clip_score(model, model_param_dict['score_clip']) train_ys_pred = model.predict(train_xs) # apply instructions indicated in the appended info train_ys_pred = VmafQualityRunner.clip_score(model, train_ys_pred) train_stats = TrainTestModel.get_stats(train_ys['label'], train_ys_pred) if logger: logger.info('Stats on training data: {}'.format( TrainTestModel.format_stats(train_stats))) # save model if output_model_filepath is not None: model.to_file(output_model_filepath) if train_ax is not None: train_content_ids = map(lambda asset: asset.content_id, train_assets) TrainTestModel.plot_scatter(train_ax, train_stats, train_content_ids) train_ax.set_xlabel('DMOS') train_ax.set_ylabel("Predicted Score") train_ax.grid() train_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format( dataset=train_dataset.dataset_name, model=model.model_id, stats=TrainTestModel.format_stats(train_stats))) # === test model on test dataset === if test_dataset is None: test_assets = None test_stats = None test_fassembler = None else: test_assets = read_dataset(test_dataset) test_fassembler = FeatureAssembler( feature_dict=feature_param.feature_dict, feature_option_dict=None, assets=test_assets, logger=logger, fifo_mode=fifo_mode, delete_workdir=True, result_store=result_store, parallelize=True, ) test_fassembler.run() test_features = test_fassembler.results test_xs = TrainTestModel.get_xs_from_results(test_features) test_ys = TrainTestModel.get_ys_from_results(test_features) test_ys_pred = model.predict(test_xs) # apply instructions indicated in the appended info test_ys_pred = VmafQualityRunner.clip_score(model, test_ys_pred) test_stats = TrainTestModel.get_stats(test_ys['label'], test_ys_pred) if logger: logger.info('Stats on testing data: {}'.format( TrainTestModel.format_stats(test_stats))) if test_ax is not None: test_content_ids = map(lambda asset: asset.content_id, test_assets) TrainTestModel.plot_scatter(test_ax, test_stats, test_content_ids) test_ax.set_xlabel('DMOS') test_ax.set_ylabel("Predicted Score") test_ax.grid() test_ax.set_title( "Dataset: {dataset}, Model: {model}\n{stats}".format( dataset=test_dataset.dataset_name, model=model.model_id, stats=TrainTestModel.format_stats(test_stats))) return train_fassembler, train_assets, train_stats, \ test_fassembler, test_assets, test_stats