def test_check_flag_column_convert_to_list_keep_numeric():
config = {"flag_column": {"advisories": 123}}
flag_dict = check_flag_column(config)
eq_(flag_dict, {"advisories": [123]})
def test_check_flag_column_wrong_format():
config = {"flag_column": "[advisories]"}
check_flag_column(config)
def test_check_flag_column_no_values():
config = {"flag_column": None}
flag_dict = check_flag_column(config)
eq_(flag_dict, {})
def test_check_flag_column_convert_to_list():
config = {"flag_column": {"advisories": "0"}}
flag_dict = check_flag_column(config)
eq_(flag_dict, {"advisories": ['0']})
def test_check_flag_column():
input_dict = {"advisory flag": ['0']}
config = {"flag_column": input_dict}
output_dict = check_flag_column(config)
eq_(input_dict, output_dict)
def test_check_flag_column_keep_numeric():
input_dict = {"advisory flag": [1, 2, 3]}
config = {"flag_column": input_dict}
output_dict = check_flag_column(config)
eq_(output_dict, {"advisory flag": [1, 2, 3]})
def test_check_flag_column_wrong_format():
config = {"flag_column": "[advisories]"}
check_flag_column(config)
def compute_and_save_predictions(config_file, output_file, feats_file):
"""
Generate predictions using the information in the config file
and save them into the given output file.
"""
logger = logging.getLogger(__name__)
# read in the main config file
config_obj = read_json_file(config_file)
config_obj = check_main_config(config_obj, context='rsmpredict')
# get the directory where the config file lives
# if this is the 'expm' directory, then go
# up one level.
configpath = dirname(config_file)
# get the input file containing the feature values
# for which we want to generate the predictions
input_features_file = locate_file(config_obj['input_features_file'], configpath)
if not input_features_file:
raise FileNotFoundError('Input file {} does not exist'.format(config_obj['input_features_file']))
# get the experiment ID
experiment_id = config_obj['experiment_id']
# get the column name that will hold the ID
id_column = config_obj['id_column']
# get the column name for human score (if any)
human_score_column = config_obj['human_score_column']
# get the column name for second human score (if any)
second_human_score_column = config_obj['second_human_score_column']
# get the column name for subgroups (if any)
subgroups = config_obj['subgroups']
# get the column names for flag columns (if any)
flag_column_dict = check_flag_column(config_obj)
# get the name for the candidate_column (if any)
candidate_column = config_obj['candidate_column']
# get the directory of the experiment
experiment_dir = locate_file(config_obj['experiment_dir'], configpath)
if not experiment_dir:
raise FileNotFoundError('The directory {} does not exist.'.format(config_obj['experiment_dir']))
else:
experiment_output_dir = normpath(join(experiment_dir, 'output'))
if not exists(experiment_output_dir):
raise FileNotFoundError('The directory {} does not contain '
'the output of an rsmtool experiment.'.format(experiment_dir))
# find all the .model files in the experiment output directory
model_files = glob.glob(join(experiment_output_dir, '*.model'))
if not model_files:
raise FileNotFoundError('The directory {} does not contain any rsmtool models.'.format(experiment_output_dir))
experiment_ids = [splitext(basename(mf))[0] for mf in model_files]
if experiment_id not in experiment_ids:
raise FileNotFoundError('{} does not contain a model for the experiment "{}". ' 'The following experiments are contained in this '
'directory: {}'.format(experiment_output_dir,
experiment_id,
experiment_ids))
# check that the directory contains outher required files
required_file_types = ['feature', 'postprocessing_params']
for file_type in required_file_types:
expected_file_name = "{}_{}.csv".format(experiment_id, file_type)
if not exists(join(experiment_output_dir, expected_file_name)):
raise FileNotFoundError('{} does not contain the required file '
'{} that was generated during the '
'original model training'.format(experiment_output_dir,
expected_file_name))
# read in the given features but make sure that the
# `id_column`, `candidate_column` and subgroups are read in as a string
logger.info('Reading features from {}'.format(input_features_file))
string_columns = [id_column, candidate_column] + subgroups
converter_dict = dict([(column, str) for column in string_columns if column])
df_input = pd.read_csv(input_features_file, converters=converter_dict)
# make sure that the columns specified in the config file actually exist
columns_to_check = [id_column] + subgroups + list(flag_column_dict.keys())
# add subgroups and the flag columns to the list of columns
# that will be added to the final file
columns_to_copy = subgroups + list(flag_column_dict.keys())
# human_score_column will be set to sc1 by default
# we only raise an error if it's set to something else.
# However, since we cannot distinguish whether the column was set
# to sc1 by default or specified as such in the config file
# we append it to output anyway as long as
# it is in the input file
if human_score_column != 'sc1' or 'sc1' in df_input.columns:
columns_to_check.append(human_score_column)
columns_to_copy.append('sc1')
if candidate_column:
columns_to_check.append(candidate_column)
columns_to_copy.append('candidate')
if second_human_score_column:
columns_to_check.append(second_human_score_column)
columns_to_copy.append('sc2')
missing_columns = set(columns_to_check).difference(df_input.columns)
if missing_columns:
raise KeyError("Columns {} from the config file "
"do not exist in the data.".format(missing_columns))
# rename all columns
df_input = rename_default_columns(df_input,
[],
id_column,
human_score_column,
second_human_score_column,
None,
None,
candidate_column=candidate_column)
# check that the id_column contains unique values
if df_input['spkitemid'].size != df_input['spkitemid'].unique().size:
raise ValueError("The data contains repeated response IDs in {}. Please make sure all response IDs are unique and re-run the tool.".format(id_column))
# now we need to pre-process these features using
# the parameters that are already stored in the
# _features.csv file.
df_feature_info = pd.read_csv(join(experiment_output_dir,
'{}_feature.csv'.format(experiment_id)),
index_col=0)
required_features = df_feature_info.index.tolist()
# ensure that all the features that are needed by the model
# are present in the input file
input_feature_columns = [c for c in df_input if c != id_column]
missing_features = set(required_features).difference(input_feature_columns)
if missing_features:
raise KeyError('{} is missing the following features: {}'.format(feats_file, missing_features))
extra_features = set(input_feature_columns).difference(required_features + [id_column])
if extra_features:
logging.warning('The following extraenous features will be ignored: {}'.format(extra_features))
# keep the required features plus the id
features_to_keep = ['spkitemid'] + required_features
# check if actually have the human scores for this data and add
# sc1 to preprocessed features for consistency with other tools
has_human_scores = 'sc1' in df_input
if has_human_scores:
features_to_keep.append('sc1')
df_features = df_input[features_to_keep]
# preprocess the feature values
logger.info('Pre-processing input features')
# first we need to filter out NaNs and any other
# weird features, the same way we did for rsmtool.
df_filtered = df_features.copy()
df_excluded = pd.DataFrame(columns=df_filtered.columns)
for feature_name in required_features:
newdf, newdf_excluded = filter_on_column(df_filtered, feature_name, 'spkitemid',
exclude_zeros=False,
exclude_zero_sd=False)
del df_filtered
df_filtered = newdf
df_excluded = pd.merge(df_excluded, newdf_excluded, how='outer')
# make sure that the remaining data frame is not empty
if len(df_filtered) == 0:
raise ValueError("There are no responses left after "
"filtering out non-numeric feature values. No analysis "
"will be run")
df_features = df_filtered.copy()
df_features_preprocessed = df_features.copy()
for feature_name in required_features:
feature_values = df_features[feature_name].values
feature_transformation = df_feature_info.loc[feature_name]['transform']
feature_weight = df_feature_info.loc[feature_name]['sign']
train_feature_mean = df_feature_info.loc[feature_name]['train_mean']
train_feature_sd = df_feature_info.loc[feature_name]['train_sd']
train_transformed_mean = df_feature_info.loc[feature_name]['train_transformed_mean']
train_transformed_sd = df_feature_info.loc[feature_name]['train_transformed_sd']
# transform the feature values and remove outliers
df_features_preprocessed[feature_name] = preprocess_feature(feature_values,
feature_name,
feature_transformation,
train_feature_mean,
train_feature_sd,
exclude_zero_sd=False)
# now standardize the feature values
df_features_preprocessed[feature_name] = (df_features_preprocessed[feature_name] - train_transformed_mean) / train_transformed_sd
# Multiply features by weight. Within the
# current SR timeline, the mean of the transformed train
# feature used to standardize test features has to be
# computed before multiplying the train feature by the weight.
df_features_preprocessed[feature_name] = df_features_preprocessed[feature_name] * feature_weight
# save the pre-processed features to disk if we were asked to
if feats_file:
logger.info('Saving pre-processed feature values to {}'.format(feats_file))
# create any directories needed for the output file
os.makedirs(dirname(feats_file), exist_ok=True)
df_features_preprocessed.to_csv(feats_file, index=False)
# now load the SKLL model to generate the predictions
model = Learner.from_file(join(experiment_output_dir, '{}.model'.format(experiment_id)))
# now generate the predictions for the features using this model
logger.info('Generating predictions')
df_predictions = predict_with_model(model, df_features_preprocessed)
# read in the post-processing parameters from disk
df_postproc_params = pd.read_csv(join(experiment_output_dir, '{}_postprocessing_params.csv'.format(experiment_id)))
trim_min = df_postproc_params['trim_min'].values[0]
trim_max = df_postproc_params['trim_max'].values[0]
h1_mean = df_postproc_params['h1_mean'].values[0]
h1_sd = df_postproc_params['h1_sd'].values[0]
train_predictions_mean = df_postproc_params['train_predictions_mean'].values[0]
train_predictions_sd = df_postproc_params['train_predictions_sd'].values[0]
# now scale the predictions
logger.info('Rescaling predictions')
scaled_predictions = (df_predictions['raw'] - train_predictions_mean) / train_predictions_sd
scaled_predictions = scaled_predictions * h1_sd + h1_mean
df_predictions['scale'] = scaled_predictions
# trim and round the predictions
logger.info('Trimming and rounding predictions')
df_predictions['raw_trim'] = trim(df_predictions['raw'], trim_min, trim_max)
df_predictions['raw_trim_round'] = np.rint(df_predictions['raw_trim']).astype('int64')
df_predictions['scale_trim'] = trim(df_predictions['scale'], trim_min, trim_max)
df_predictions['scale_trim_round'] = np.rint(df_predictions['scale_trim']).astype('int64')
# add back the columns that we were requested to copy if any
if columns_to_copy:
df_predictions_with_metadata = pd.merge(df_predictions,
df_input[['spkitemid'] + columns_to_copy])
assert(len(df_predictions) == len(df_predictions_with_metadata))
else:
df_predictions_with_metadata = df_predictions.copy()
# create any directories needed for the output file
os.makedirs(dirname(output_file), exist_ok=True)
# save the predictions to disk
logger.info('Saving predictions to {}'.format(output_file))
df_predictions_with_metadata.to_csv(output_file, index=False)
# save excluded responses to disk
if not df_excluded.empty:
excluded_output_file = '{}_excluded_responses{}'.format(*splitext(output_file))
logger.info('Saving excluded responses to {}'.format(excluded_output_file))
df_excluded.to_csv(excluded_output_file, index=False)
def test_check_flag_column_convert_to_list():
config = {"flag_column": {"advisories": "0"}}
flag_dict = check_flag_column(config)
eq_(flag_dict, {"advisories": ['0']})
def test_check_flag_column_convert_to_list_keep_numeric():
config = {"flag_column": {"advisories": 123}}
flag_dict = check_flag_column(config)
eq_(flag_dict, {"advisories": [123]})
def test_check_flag_column_no_values():
config = {"flag_column": None}
flag_dict = check_flag_column(config)
eq_(flag_dict, {})
def test_check_flag_column_keep_numeric():
input_dict = {"advisory flag": [1, 2, 3]}
config = {"flag_column": input_dict}
output_dict = check_flag_column(config)
eq_(output_dict, {"advisory flag": [1, 2, 3]})
def test_check_flag_column():
input_dict = {"advisory flag": ['0']}
config = {"flag_column": input_dict}
output_dict = check_flag_column(config)
eq_(input_dict, output_dict)
def run_evaluation(config_file, output_dir):
"""
Run RSMTool evaluation experiment using the given configuration
file and generate all evaluation outputs in the given directory.
"""
logger = logging.getLogger(__name__)
# create the 'output' and the 'figure' sub-directories
# where all the experiment output such as the CSV files
# and the box plots will be saved
csvdir = abspath(join(output_dir, 'output'))
figdir = abspath(join(output_dir, 'figure'))
reportdir = abspath(join(output_dir, 'report'))
os.makedirs(csvdir, exist_ok=True)
os.makedirs(figdir, exist_ok=True)
os.makedirs(reportdir, exist_ok=True)
# load the information from the config file
# read in the main config file
config_obj = read_json_file(config_file)
config_obj = check_main_config(config_obj, context='rsmeval')
# get the directory where the config file lives
# if this is the 'expm' directory, then go
# up one level.
configpath = dirname(config_file)
# get the experiment ID
experiment_id = config_obj['experiment_id']
# get the description
description = config_obj['description']
# get the column name for the labels for the training and testing data
human_score_column = config_obj['human_score_column']
system_score_column = config_obj['system_score_column']
# get the name of the optional column that
# contains the second human score
second_human_score_column = config_obj['second_human_score_column']
# if the human score column is the same as the
# second human score column, raise an error
if human_score_column == second_human_score_column:
raise ValueError("'human_score_column' and "
"'second_human_score_column' "
"cannot have the same value.")
# get the column name that will hold the ID for
# both the training and the test data
id_column = config_obj['id_column']
# get the specified trim min and max, if any
# and make sure they are numeric
spec_trim_min, spec_trim_max = get_trim_min_max(config_obj)
# get the subgroups if any
subgroups = config_obj.get('subgroups')
# get the candidate column if any and convert it to string
candidate_column = config_obj['candidate_column']
general_report_sections = config_obj['general_sections']
# get any special sections that the user might have specified
special_report_sections = config_obj['special_sections']
# get any custom sections and locate them to make sure
# that they exist, otherwise raise an exception
custom_report_section_paths = config_obj['custom_sections']
if custom_report_section_paths:
logger.info('Locating custom report sections')
custom_report_sections = locate_custom_sections(
custom_report_section_paths, configpath)
else:
custom_report_sections = []
section_order = config_obj['section_order']
# check all sections values and order and get the
# ordered list of notebook files
chosen_notebook_files = get_ordered_notebook_files(general_report_sections,
special_report_sections,
custom_report_sections,
section_order,
subgroups,
model_type=None,
context='rsmeval')
# are we excluding zero scores?
exclude_zero_scores = config_obj['exclude_zero_scores']
# if we are excluding zero scores but trim_min
# is set to 0, then we need to warn the user
if exclude_zero_scores and spec_trim_min == 0:
logger.warning("'exclude_zero_scores' is set to True but "
" 'trim_min' is set to 0. This may cause "
" unexpected behavior.")
# are we filtering on any other columns?
flag_column_dict = check_flag_column(config_obj)
# do we have the training set predictions and human scores CSV file
scale_with = config_obj.get('scale_with')
# scale_with can be one of the following:
# (a) None : the predictions are assumed to be 'raw' and should be used as is
# when computing the metrics; the names for the final columns are
# 'raw', 'raw_trim' and 'raw_trim_round'.
# (b) 'asis' : the predictions are assumed to be pre-scaled and should be used as is
# when computing the metrics; the names for the final columns are
# 'scale', 'scale_trim' and 'scale_trim_round'.
# (c) a CSV file : the predictions are assumed to be 'raw' and should be scaled
# before computing the metrics; the names for the final columns are
# 'scale', 'scale_trim' and 'scale_trim_round'.
# we need to scale if and only if a CSV file is specified
do_scaling = (scale_with != None and scale_with != 'asis')
# use scaled predictions for the analyses unless
# we were told not to
use_scaled_predictions = (scale_with != None)
# log an appropriate message
if scale_with is None:
message = ('Assuming given system predictions '
'are unscaled and will be used as such.')
elif scale_with == 'asis':
message = ('Assuming given system predictions '
'are already scaled and will be used as such.')
else:
message = ('Assuming given system predictions '
'are unscaled and will be scaled before use.')
logger.info(message)
# load the predictions from disk and make sure that the `id_column`
# is read in as a string
predictions_file_location = locate_file(config_obj['predictions_file'],
configpath)
if not predictions_file_location:
raise FileNotFoundError('Error: Predictions file {} '
'not found.\n'.format(
config_obj['predictions_file']))
else:
logger.info(
'Reading predictions: {}'.format(predictions_file_location))
string_columns = [id_column, candidate_column] + subgroups
converter_dict = dict([(column, str) for column in string_columns
if column])
df_pred = pd.read_csv(predictions_file_location,
converters=converter_dict)
# make sure that the columns specified in the config file actually exist
missing_columns = set([id_column, human_score_column,
system_score_column]).difference(df_pred.columns)
if missing_columns:
raise KeyError('Columns {} from the config file do not exist '
'in the predictions file.'.format(missing_columns))
df_pred = rename_default_columns(df_pred, [], id_column,
human_score_column,
second_human_score_column, None,
system_score_column, candidate_column)
# check that the id_column contains unique values
if df_pred['spkitemid'].size != df_pred['spkitemid'].unique().size:
raise ValueError("The data contains duplicate response IDs "
"in '{}'. Please make sure all response IDs "
"are unique and re-run the tool.".format(id_column))
df_pred = check_subgroups(df_pred, subgroups)
# filter out the responses based on flag columns
(df_responses_with_requested_flags,
df_responses_with_excluded_flags) = filter_on_flag_columns(
df_pred, flag_column_dict)
# filter out rows that have non-numeric or zero human scores
df_filtered, df_excluded = filter_on_column(
df_responses_with_requested_flags,
'sc1',
'spkitemid',
exclude_zeros=exclude_zero_scores)
# make sure that the remaining data frame is not empty
if len(df_filtered) == 0:
raise ValueError("No responses remaining after filtering out "
"non-numeric human scores. No further analysis "
"can be run. ")
# Change all non-numeric machine scores in excluded
# data to NaNs for consistency with rsmtool.
# NOTE: This will *not* work if *all* of the values
# in column are non-numeric. This is a known bug in
# pandas: https://github.com/pydata/pandas/issues/9589
# Therefore, we need add an additional check after this.
df_excluded['raw'] = pd.to_numeric(df_excluded['raw'],
errors='coerce').astype(float)
# filter out the non-numeric machine scores from the rest of the data
newdf, newdf_excluded = filter_on_column(df_filtered,
'raw',
'spkitemid',
exclude_zeros=False)
del df_filtered
df_filtered_pred = newdf
# make sure that the remaining data frame is not empty
if len(df_filtered_pred) == 0:
raise ValueError("No responses remaining after filtering out "
"non-numeric machine scores. No further analysis "
"can be run. ")
df_excluded = pd.merge(df_excluded, newdf_excluded, how='outer')
# set default values for scaling
scale_pred_mean = 0
scale_pred_sd = 1
scale_human_mean = 0
scale_human_sd = 1
if do_scaling:
scale_file_location = locate_file(scale_with, configpath)
if not scale_file_location:
raise FileNotFoundError(
'Error: scaling file {} not found.\n'.format(scale_with))
else:
logger.info('Reading scaling file: {}'.format(scale_file_location))
df_scale_with = pd.read_csv(scale_file_location)
if 'sc1' not in df_scale_with.columns and 'prediction' not in df_scale_with.columns:
raise KeyError(
'The CSV file specified for scaling ',
'must have the "prediction" and the "sc1" '
'columns.')
else:
scale_pred_mean, scale_pred_sd = (
df_scale_with['prediction'].mean(),
df_scale_with['prediction'].std())
scale_human_mean, scale_human_sd = (df_scale_with['sc1'].mean(),
df_scale_with['sc1'].std())
logger.info('Processing predictions')
df_pred_processed = process_predictions(df_filtered_pred, scale_pred_mean,
scale_pred_sd, scale_human_mean,
scale_human_sd, spec_trim_min,
spec_trim_max)
if not scale_with:
expected_score_types = ['raw', 'raw_trim', 'raw_trim_round']
elif scale_with == 'asis':
expected_score_types = ['scale', 'scale_trim', 'scale_trim_round']
else:
expected_score_types = [
'raw', 'raw_trim', 'raw_trim_round', 'scale', 'scale_trim',
'scale_trim_round'
]
# extract separated data frames that we will write out
# as separate files
not_other_columns = set()
prediction_columns = ['spkitemid', 'sc1'] + expected_score_types
df_predictions_only = df_pred_processed[prediction_columns]
not_other_columns.update(prediction_columns)
metadata_columns = ['spkitemid'] + subgroups
if candidate_column:
metadata_columns.append('candidate')
df_test_metadata = df_filtered_pred[metadata_columns]
not_other_columns.update(metadata_columns)
df_test_human_scores = pd.DataFrame()
human_score_columns = ['spkitemid', 'sc1', 'sc2']
if second_human_score_column and 'sc2' in df_filtered_pred:
df_test_human_scores = df_filtered_pred[human_score_columns].copy()
not_other_columns.update(['sc2'])
# filter out any non-numeric values nows
# as well as zeros, if we were asked to
df_test_human_scores['sc2'] = pd.to_numeric(
df_test_human_scores['sc2'], errors='coerce').astype(float)
if exclude_zero_scores:
df_test_human_scores['sc2'] = df_test_human_scores['sc2'].replace(
0, np.nan)
# remove 'spkitemid' from `not_other_columns`
# because we want that in the other columns
# data frame
not_other_columns.remove('spkitemid')
# extract all of the other columns in the predictions file
other_columns = [
column for column in df_filtered_pred.columns
if column not in not_other_columns
]
df_pred_other_columns = df_filtered_pred[other_columns]
logger.info('Saving pre-processed predictions and the metadata to disk')
write_experiment_output([
df_predictions_only, df_test_metadata, df_pred_other_columns,
df_test_human_scores, df_excluded, df_responses_with_excluded_flags
], [
'pred_processed', 'test_metadata', 'test_other_columns',
'test_human_scores', 'test_excluded_responses',
'test_responses_with_excluded_flags'
], experiment_id, csvdir)
# do the data composition stats
(df_test_excluded_analysis, df_data_composition,
data_composition_by_group_dict
) = run_data_composition_analyses_for_rsmeval(
df_test_metadata,
df_excluded,
subgroups,
candidate_column,
exclude_zero_scores=exclude_zero_scores)
write_experiment_output([df_test_excluded_analysis, df_data_composition],
['test_excluded_composition', 'data_composition'],
experiment_id, csvdir)
# write the results of data composition analysis by group
if subgroups:
for group in subgroups:
write_experiment_output([data_composition_by_group_dict[group]],
['data_composition_by_{}'.format(group)],
experiment_id, csvdir)
# run the analyses on the predictions of the modelx`
logger.info('Running analyses on predictions')
(df_human_machine_eval, df_human_machine_eval_short, df_human_human_eval,
eval_by_group_dict, df_degradation, df_confmatrix,
df_score_dist) = run_prediction_analyses(
df_predictions_only,
df_test_metadata,
df_test_human_scores,
subgroups,
second_human_score_column,
exclude_zero_scores=exclude_zero_scores,
use_scaled_predictions=use_scaled_predictions)
write_experiment_output([
df_human_machine_eval, df_human_machine_eval_short,
df_human_human_eval, df_degradation, df_confmatrix, df_score_dist
], [
'eval', 'eval_short', 'consistency', 'degradation', 'confMatrix',
'score_dist'
],
experiment_id,
csvdir,
reset_index=True)
# if we are using subgroups, then write out the subgroup
# specific output and include the by group section
# in the final report
if subgroups:
for group in subgroups:
eval_by_group, consistency_by_group = eval_by_group_dict[group]
write_experiment_output([eval_by_group, consistency_by_group], [
'eval_by_{}'.format(group), 'consistency_by_{}'.format(group)
],
experiment_id,
csvdir,
reset_index=True)
# generate the report
logger.info('Starting report generation')
create_report(experiment_id,
description,
'',
'',
'',
predictions_file_location,
csvdir,
figdir,
subgroups,
None,
second_human_score_column,
chosen_notebook_files,
exclude_zero_scores=exclude_zero_scores,
use_scaled_predictions=use_scaled_predictions)
def compute_and_save_predictions(config_file, output_file, feats_file):
"""
Generate predictions using the information in the config file
and save them into the given output file.
"""
logger = logging.getLogger(__name__)
# read in the main config file
config_obj = read_json_file(config_file)
config_obj = check_main_config(config_obj, context='rsmpredict')
# get the directory where the config file lives
# if this is the 'expm' directory, then go
# up one level.
configpath = dirname(config_file)
# get the input file containing the feature values
# for which we want to generate the predictions
input_features_file = locate_file(config_obj['input_features_file'],
configpath)
if not input_features_file:
raise FileNotFoundError('Input file {} does not exist'.format(
config_obj['input_features_file']))
# get the experiment ID
experiment_id = config_obj['experiment_id']
# get the column name that will hold the ID
id_column = config_obj['id_column']
# get the column name for human score (if any)
human_score_column = config_obj['human_score_column']
# get the column name for second human score (if any)
second_human_score_column = config_obj['second_human_score_column']
# get the column name for subgroups (if any)
subgroups = config_obj['subgroups']
# get the column names for flag columns (if any)
flag_column_dict = check_flag_column(config_obj)
# get the name for the candidate_column (if any)
candidate_column = config_obj['candidate_column']
# get the directory of the experiment
experiment_dir = locate_file(config_obj['experiment_dir'], configpath)
if not experiment_dir:
raise FileNotFoundError('The directory {} does not exist.'.format(
config_obj['experiment_dir']))
else:
experiment_output_dir = normpath(join(experiment_dir, 'output'))
if not exists(experiment_output_dir):
raise FileNotFoundError(
'The directory {} does not contain '
'the output of an rsmtool experiment.'.format(experiment_dir))
# find all the .model files in the experiment output directory
model_files = glob.glob(join(experiment_output_dir, '*.model'))
if not model_files:
raise FileNotFoundError(
'The directory {} does not contain any rsmtool models.'.format(
experiment_output_dir))
experiment_ids = [splitext(basename(mf))[0] for mf in model_files]
if experiment_id not in experiment_ids:
raise FileNotFoundError(
'{} does not contain a model for the experiment "{}". '
'The following experiments are contained in this '
'directory: {}'.format(experiment_output_dir, experiment_id,
experiment_ids))
# check that the directory contains outher required files
required_file_types = ['feature', 'postprocessing_params']
for file_type in required_file_types:
expected_file_name = "{}_{}.csv".format(experiment_id, file_type)
if not exists(join(experiment_output_dir, expected_file_name)):
raise FileNotFoundError('{} does not contain the required file '
'{} that was generated during the '
'original model training'.format(
experiment_output_dir,
expected_file_name))
# read in the given features but make sure that the
# `id_column`, `candidate_column` and subgroups are read in as a string
logger.info('Reading features from {}'.format(input_features_file))
string_columns = [id_column, candidate_column] + subgroups
converter_dict = dict([(column, str) for column in string_columns
if column])
df_input = pd.read_csv(input_features_file, converters=converter_dict)
# make sure that the columns specified in the config file actually exist
columns_to_check = [id_column] + subgroups + list(flag_column_dict.keys())
# add subgroups and the flag columns to the list of columns
# that will be added to the final file
columns_to_copy = subgroups + list(flag_column_dict.keys())
# human_score_column will be set to sc1 by default
# we only raise an error if it's set to something else.
# However, since we cannot distinguish whether the column was set
# to sc1 by default or specified as such in the config file
# we append it to output anyway as long as
# it is in the input file
if human_score_column != 'sc1' or 'sc1' in df_input.columns:
columns_to_check.append(human_score_column)
columns_to_copy.append('sc1')
if candidate_column:
columns_to_check.append(candidate_column)
columns_to_copy.append('candidate')
if second_human_score_column:
columns_to_check.append(second_human_score_column)
columns_to_copy.append('sc2')
missing_columns = set(columns_to_check).difference(df_input.columns)
if missing_columns:
raise KeyError("Columns {} from the config file "
"do not exist in the data.".format(missing_columns))
# rename all columns
df_input = rename_default_columns(df_input, [],
id_column,
human_score_column,
second_human_score_column,
None,
None,
candidate_column=candidate_column)
# check that the id_column contains unique values
if df_input['spkitemid'].size != df_input['spkitemid'].unique().size:
raise ValueError(
"The data contains repeated response IDs in {}. Please make sure all response IDs are unique and re-run the tool."
.format(id_column))
# now we need to pre-process these features using
# the parameters that are already stored in the
# _features.csv file.
df_feature_info = pd.read_csv(join(experiment_output_dir,
'{}_feature.csv'.format(experiment_id)),
index_col=0)
required_features = df_feature_info.index.tolist()
# ensure that all the features that are needed by the model
# are present in the input file
input_feature_columns = [c for c in df_input if c != id_column]
missing_features = set(required_features).difference(input_feature_columns)
if missing_features:
raise KeyError('{} is missing the following features: {}'.format(
feats_file, missing_features))
extra_features = set(input_feature_columns).difference(required_features +
[id_column])
if extra_features:
logging.warning(
'The following extraenous features will be ignored: {}'.format(
extra_features))
# keep the required features plus the id
features_to_keep = ['spkitemid'] + required_features
# check if actually have the human scores for this data and add
# sc1 to preprocessed features for consistency with other tools
has_human_scores = 'sc1' in df_input
if has_human_scores:
features_to_keep.append('sc1')
df_features = df_input[features_to_keep]
# preprocess the feature values
logger.info('Pre-processing input features')
# first we need to filter out NaNs and any other
# weird features, the same way we did for rsmtool.
df_filtered = df_features.copy()
df_excluded = pd.DataFrame(columns=df_filtered.columns)
for feature_name in required_features:
newdf, newdf_excluded = filter_on_column(df_filtered,
feature_name,
'spkitemid',
exclude_zeros=False,
exclude_zero_sd=False)
del df_filtered
df_filtered = newdf
df_excluded = pd.merge(df_excluded, newdf_excluded, how='outer')
# make sure that the remaining data frame is not empty
if len(df_filtered) == 0:
raise ValueError(
"There are no responses left after "
"filtering out non-numeric feature values. No analysis "
"will be run")
df_features = df_filtered.copy()
df_features_preprocessed = df_features.copy()
for feature_name in required_features:
feature_values = df_features[feature_name].values
feature_transformation = df_feature_info.loc[feature_name]['transform']
feature_weight = df_feature_info.loc[feature_name]['sign']
train_feature_mean = df_feature_info.loc[feature_name]['train_mean']
train_feature_sd = df_feature_info.loc[feature_name]['train_sd']
train_transformed_mean = df_feature_info.loc[feature_name][
'train_transformed_mean']
train_transformed_sd = df_feature_info.loc[feature_name][
'train_transformed_sd']
# transform the feature values and remove outliers
df_features_preprocessed[feature_name] = preprocess_feature(
feature_values,
feature_name,
feature_transformation,
train_feature_mean,
train_feature_sd,
exclude_zero_sd=False)
# now standardize the feature values
df_features_preprocessed[feature_name] = (
df_features_preprocessed[feature_name] -
train_transformed_mean) / train_transformed_sd
# Multiply features by weight. Within the
# current SR timeline, the mean of the transformed train
# feature used to standardize test features has to be
# computed before multiplying the train feature by the weight.
df_features_preprocessed[feature_name] = df_features_preprocessed[
feature_name] * feature_weight
# save the pre-processed features to disk if we were asked to
if feats_file:
logger.info(
'Saving pre-processed feature values to {}'.format(feats_file))
# create any directories needed for the output file
os.makedirs(dirname(feats_file), exist_ok=True)
df_features_preprocessed.to_csv(feats_file, index=False)
# now load the SKLL model to generate the predictions
model = Learner.from_file(
join(experiment_output_dir, '{}.model'.format(experiment_id)))
# now generate the predictions for the features using this model
logger.info('Generating predictions')
df_predictions = predict_with_model(model, df_features_preprocessed)
# read in the post-processing parameters from disk
df_postproc_params = pd.read_csv(
join(experiment_output_dir,
'{}_postprocessing_params.csv'.format(experiment_id)))
trim_min = df_postproc_params['trim_min'].values[0]
trim_max = df_postproc_params['trim_max'].values[0]
h1_mean = df_postproc_params['h1_mean'].values[0]
h1_sd = df_postproc_params['h1_sd'].values[0]
train_predictions_mean = df_postproc_params[
'train_predictions_mean'].values[0]
train_predictions_sd = df_postproc_params['train_predictions_sd'].values[0]
# now scale the predictions
logger.info('Rescaling predictions')
scaled_predictions = (df_predictions['raw'] -
train_predictions_mean) / train_predictions_sd
scaled_predictions = scaled_predictions * h1_sd + h1_mean
df_predictions['scale'] = scaled_predictions
# trim and round the predictions
logger.info('Trimming and rounding predictions')
df_predictions['raw_trim'] = trim(df_predictions['raw'], trim_min,
trim_max)
df_predictions['raw_trim_round'] = np.rint(
df_predictions['raw_trim']).astype('int64')
df_predictions['scale_trim'] = trim(df_predictions['scale'], trim_min,
trim_max)
df_predictions['scale_trim_round'] = np.rint(
df_predictions['scale_trim']).astype('int64')
# add back the columns that we were requested to copy if any
if columns_to_copy:
df_predictions_with_metadata = pd.merge(
df_predictions, df_input[['spkitemid'] + columns_to_copy])
assert (len(df_predictions) == len(df_predictions_with_metadata))
else:
df_predictions_with_metadata = df_predictions.copy()
# create any directories needed for the output file
os.makedirs(dirname(output_file), exist_ok=True)
# save the predictions to disk
logger.info('Saving predictions to {}'.format(output_file))
df_predictions_with_metadata.to_csv(output_file, index=False)
# save excluded responses to disk
if not df_excluded.empty:
excluded_output_file = '{}_excluded_responses{}'.format(
*splitext(output_file))
logger.info(
'Saving excluded responses to {}'.format(excluded_output_file))
df_excluded.to_csv(excluded_output_file, index=False)