def calculate_and_log_val_metrics(nn_model,
context_sensitive_val_subset,
context_free_val,
prediction_mode=PREDICTION_MODE_FOR_TESTS):
val_metrics = dict()
val_metrics[
'context_sensitive_perplexity'] = calculate_model_mean_perplexity(
nn_model, context_sensitive_val_subset)
val_metrics['context_free_perplexity'] = calculate_model_mean_perplexity(
nn_model, context_free_val)
val_metrics[
'unigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x,
nn_model,
ngram_len=1,
mode=prediction_mode)
val_metrics['bigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x,
nn_model,
ngram_len=2,
mode=prediction_mode)
_logger.info('Current val context-sensitive perplexity: %s' %
val_metrics['context_sensitive_perplexity'])
_logger.info('Current val context-free perplexity: %s' %
val_metrics['context_free_perplexity'])
_logger.info('Current val distinctness: uni=%s, bi=%s' %
(val_metrics['unigram_distinctness'],
val_metrics['bigram_distinctness']))
return val_metrics
def calculate_and_log_val_metrics(nn_model,
context_sensitive_val,
context_free_val,
prediction_mode=PREDICTION_MODE_FOR_TESTS,
calculate_ngram_distance=True):
metric_name_to_value = {
'context_free_perplexity':
calculate_model_mean_perplexity(nn_model, context_free_val),
'context_sensitive_perplexity':
calculate_model_mean_perplexity(nn_model, context_sensitive_val)
}
if calculate_ngram_distance:
for metric_name, ngram_len in [('unigram_distinctness', 1),
('bigram_distinctness', 2)]:
metric_name_to_value[
metric_name] = calculate_response_ngram_distinctness(
context_sensitive_val.x,
nn_model,
ngram_len=ngram_len,
mode=prediction_mode,
condition_ids=context_sensitive_val.condition_ids)
for metric_name, metric_value in metric_name_to_value.items():
_logger.info('Val set {}: {:.3f}'.format(metric_name, metric_value))
return metric_name_to_value
def calc_perplexity_metrics(nn_model, eval_datasets):
return {
'ppl_cs_test':
calculate_model_mean_perplexity(nn_model, eval_datasets.cs_test),
'ppl_cs_test_not_conditioned':
calculate_model_mean_perplexity(nn_model, _make_non_conditioned(eval_datasets.cs_test)),
'ppl_cs_test_one_condition':
calculate_model_mean_perplexity(nn_model, eval_datasets.cs_test_one_condition),
'ppl_cs_test_one_condition_not_conditioned':
calculate_model_mean_perplexity(nn_model, _make_non_conditioned(eval_datasets.cs_test_one_condition)),
'ppl_cf_validation':
calculate_model_mean_perplexity(nn_model, eval_datasets.cf_validation)
}
def calc_perplexity_by_condition_metrics(nn_model, train):
for condition, condition_id in nn_model.condition_to_index.items():
if condition == DEFAULT_CONDITION:
continue
dataset_with_conditions = _slice_condition_data(train, condition_id)
if not dataset_with_conditions.x.size:
_logger.warn('No dataset samples found with the given condition "%s", skipping metrics.' % condition)
continue
ppl_non_conditioned = calculate_model_mean_perplexity(nn_model, _make_non_conditioned(dataset_with_conditions))
ppl_conditioned = calculate_model_mean_perplexity(nn_model, dataset_with_conditions)
yield condition, (ppl_non_conditioned, ppl_conditioned)
def calc_perplexity_metrics(nn_model, train_subset, subset_with_conditions, validation):
ppl_non_conditioned_train_subset = calculate_model_mean_perplexity(nn_model, _make_non_conditioned(train_subset))
ppl_train_subset = calculate_model_mean_perplexity(nn_model, train_subset)
ppl_non_conditioned_subset_with_conditions = calculate_model_mean_perplexity(
nn_model, _make_non_conditioned(subset_with_conditions))
ppl_subset_with_conditions = calculate_model_mean_perplexity(nn_model, subset_with_conditions)
ppl_validation = calculate_model_mean_perplexity(nn_model, validation)
return {
'perplexity_train_subset_no_cond': ppl_non_conditioned_train_subset,
'perplexity_train_subset': ppl_train_subset,
'perplexity_subset_with_conditions_no_cond': ppl_non_conditioned_subset_with_conditions,
'perplexity_subset_with_conditions': ppl_subset_with_conditions,
'perplexity_validation': ppl_validation,
}
def _calc_and_save_train_metrics(nn_model, train_subset, avg_loss):
train_metrics = dict()
train_metrics['train_perplexity'] = calculate_model_mean_perplexity(nn_model, train_subset)
train_metrics['train_loss'] = avg_loss
save_metrics(train_metrics)
_logger.info('Current train perplexity: %s' % train_metrics['train_perplexity'])
return train_metrics
def calculate_and_log_val_metrics(nn_model,
context_sensitive_val_subset,
context_free_val,
prediction_mode=PREDICTION_MODE_FOR_TESTS):
val_metrics = dict()
val_metrics['context_sensitive_perplexity'] = calculate_model_mean_perplexity(nn_model,
context_sensitive_val_subset)
val_metrics['context_free_perplexity'] = calculate_model_mean_perplexity(nn_model, context_free_val)
val_metrics['unigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x, nn_model, ngram_len=1, mode=prediction_mode)
val_metrics['bigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x, nn_model, ngram_len=2, mode=prediction_mode)
_logger.info('Current val context-sensitive perplexity: %s' % val_metrics['context_sensitive_perplexity'])
_logger.info('Current val context-free perplexity: %s' % val_metrics['context_free_perplexity'])
_logger.info('Current val distinctness: uni=%s, bi=%s' % (val_metrics['unigram_distinctness'],
val_metrics['bigram_distinctness']))
return val_metrics
def calc_perplexity_by_condition_metrics(nn_model, train):
for condition, condition_id in nn_model.condition_to_index.items():
if condition == DEFAULT_CONDITION:
continue
dataset_with_conditions = _slice_condition_data(train, condition_id)
if not dataset_with_conditions.x.size:
_logger.warn(
'No dataset samples found with the given condition "%s", skipping metrics.'
% condition)
continue
ppl_non_conditioned = calculate_model_mean_perplexity(
nn_model, _make_non_conditioned(dataset_with_conditions))
ppl_conditioned = calculate_model_mean_perplexity(
nn_model, dataset_with_conditions)
yield condition, (ppl_non_conditioned, ppl_conditioned)
def calc_perplexity_metrics(nn_model, train_subset, subset_with_conditions,
validation):
ppl_non_conditioned_train_subset = calculate_model_mean_perplexity(
nn_model, _make_non_conditioned(train_subset))
ppl_train_subset = calculate_model_mean_perplexity(nn_model, train_subset)
ppl_non_conditioned_subset_with_conditions = calculate_model_mean_perplexity(
nn_model, _make_non_conditioned(subset_with_conditions))
ppl_subset_with_conditions = calculate_model_mean_perplexity(
nn_model, subset_with_conditions)
ppl_validation = calculate_model_mean_perplexity(nn_model, validation)
return {
'perplexity_train_subset_no_cond': ppl_non_conditioned_train_subset,
'perplexity_train_subset': ppl_train_subset,
'perplexity_subset_with_conditions_no_cond':
ppl_non_conditioned_subset_with_conditions,
'perplexity_subset_with_conditions': ppl_subset_with_conditions,
'perplexity_validation': ppl_validation,
}
def calc_perplexity_for_conditions(nn_model, dataset):
cond_to_ppl_conditioned, cond_to_ppl_not_conditioned = {}, {}
for condition, condition_id in nn_model.condition_to_index.items():
if condition == DEFAULT_CONDITION:
continue
dataset_with_conditions = _slice_condition_data(dataset, condition_id)
if not dataset_with_conditions.x.size:
_logger.warning(
'No dataset samples found with the given condition "{}", skipping metrics.'.format(condition))
continue
cond_to_ppl_conditioned[condition] = \
calculate_model_mean_perplexity(nn_model, _make_non_conditioned(dataset_with_conditions))
cond_to_ppl_not_conditioned[condition] = \
calculate_model_mean_perplexity(nn_model, dataset_with_conditions)
return cond_to_ppl_conditioned, cond_to_ppl_not_conditioned
def calculate_and_log_val_metrics(nn_model,
context_sensitive_val_subset,
context_free_val,
prediction_mode=PREDICTION_MODE_FOR_TESTS):
val_metrics = dict()
val_metrics['context_free_perplexity'] = calculate_model_mean_perplexity(
nn_model, context_free_val)
_logger.info('Current val context-free perplexity: {0:.2f}'.format(
val_metrics['context_free_perplexity']))
val_metrics['context_sensitive_perplexity'] = \
calculate_model_mean_perplexity(nn_model, context_sensitive_val_subset)
_logger.info('Current val context-sensitive perplexity: {0:.2f}'.format(
val_metrics['context_sensitive_perplexity']))
val_metrics[
'unigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x,
nn_model,
ngram_len=1,
mode=prediction_mode,
condition_ids=context_sensitive_val_subset.condition_ids)
val_metrics['bigram_distinctness'] = calculate_response_ngram_distinctness(
context_sensitive_val_subset.x,
nn_model,
ngram_len=2,
mode=prediction_mode,
condition_ids=context_sensitive_val_subset.condition_ids)
_logger.info('Current val distinctness: uni={0:.3f}, bi={1:.3f}'.format(
val_metrics['unigram_distinctness'],
val_metrics['bigram_distinctness']))
return val_metrics
