def check_print_model_weights(task='classification'):
# create some simple classification or regression data
if task == 'classification':
train_fs, _ = make_classification_data(train_test_ratio=0.8)
else:
train_fs, _, _ = make_regression_data(num_features=4,
train_test_ratio=0.8)
# now train the appropriate model
if task == 'classification':
learner = Learner('LogisticRegression')
learner.train(train_fs)
else:
learner = Learner('LinearRegression')
learner.train(train_fs, grid_objective='pearson')
# now save the model to disk
model_file = join(_my_dir, 'output',
'test_print_model_weights.model')
learner.save(model_file)
# now call print_model_weights main() and capture the output
print_model_weights_cmd = [model_file]
err = ''
try:
old_stderr = sys.stderr
old_stdout = sys.stdout
sys.stderr = mystderr = StringIO()
sys.stdout = mystdout = StringIO()
pmw.main(print_model_weights_cmd)
out = mystdout.getvalue()
err = mystderr.getvalue()
finally:
sys.stderr = old_stderr
sys.stdout = old_stdout
print(err)
# now parse the output of the print_model_weight command
# and get the intercept and the feature values
if task == 'classification':
lines_to_parse = [l for l in out.split('\n')[1:] if l]
intercept = safe_float(lines_to_parse[0].split('\t')[0])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[2], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_[0])
assert_allclose(learner.model.coef_[0], feature_values)
else:
lines_to_parse = [l for l in out.split('\n') if l]
intercept = safe_float(lines_to_parse[0].split('=')[1])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[1], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_, feature_values)
def compute_eval_from_predictions(examples_file, predictions_file,
metric_names):
"""
Compute evaluation metrics from prediction files after you have run an
experiment.
:param examples_file: a SKLL examples file (in .jsonlines or other format)
:param predictions_file: a SKLL predictions output TSV file with id
and prediction column names
:param metric_names: a list of SKLL metric names
(e.g., [pearson, unweighted_kappa])
:returns: a dictionary from metrics names to values
"""
# read gold standard labels
data = Reader.for_path(examples_file).read()
gold = dict(zip(data.ids, data.labels))
# read predictions
pred = {}
with open(predictions_file) as pred_file:
reader = csv.reader(pred_file, dialect=csv.excel_tab)
next(reader) # skip header
for row in reader:
pred[row[0]] = safe_float(row[1])
# make a sorted list of example ids in order to match up
# labels and predictions
if set(gold.keys()) != set(pred.keys()):
raise ValueError('The example and prediction IDs do not match.')
example_ids = sorted(gold.keys())
res = {}
for metric_name in metric_names:
score = use_score_func(metric_name,
[gold[ex_id] for ex_id in example_ids],
[pred[ex_id] for ex_id in example_ids])
res[metric_name] = score
return res
def compute_eval_from_predictions(examples_file, predictions_file,
metric_names):
'''
Compute evaluation metrics from prediction files after you have run an
experiment.
:param examples_file: a SKLL examples file (in .jsonlines or other format)
:param predictions_file: a SKLL predictions output TSV file with id
and prediction column names
:param metric_names: a list of SKLL metric names
(e.g., [pearson, unweighted_kappa])
:returns: a dictionary from metrics names to values
'''
# read gold standard labels
data = load_examples(examples_file)
gold = dict(zip(data.ids, data.classes))
# read predictions
pred = {}
with open(predictions_file) as pred_file:
reader = csv.reader(pred_file, dialect=csv.excel_tab)
next(reader) # skip header
for row in reader:
pred[row[0]] = safe_float(row[1])
# make a sorted list of example ids in order to match up
# labels and predictions
if set(gold.keys()) != set(pred.keys()):
raise ValueError('The example and prediction IDs do not match.')
example_ids = sorted(gold.keys())
res = {}
for metric_name in metric_names:
score = use_score_func(metric_name,
[gold[ex_id] for ex_id in example_ids],
[pred[ex_id] for ex_id in example_ids])
res[metric_name] = score
return res
def compute_eval_from_predictions(examples_file,
predictions_file,
metric_names,
prediction_method=None):
"""
Compute evaluation metrics from prediction files after you have run an
experiment.
Parameters
----------
examples_file: str
Path to a SKLL examples file (in .jsonlines or other format).
predictions_file: str
Path to a SKLL predictions output TSV file with id and prediction column names.
metric_names: list of str
A list of SKLL metric names (e.g., [pearson, unweighted_kappa]).
prediction_method: str or None
Indicates how to get a single class prediction from the probabilities. Currently
supported options are "highest", which selects the class with the highest
probability, and "expected_value", which calculates an expected value over
integer classes and rounds to the nearest int. If predictions file does not
contain probabilities, this should be set to None.
Returns
-------
dict
Maps metrics names to corresponding values.
Raises
------
ValueError
If the requested prediction method is 'expected_value' but the class names can't
be converted to ints.
"""
# read gold standard labels
data = Reader.for_path(examples_file).read()
gold = dict(zip(data.ids, data.labels))
# read predictions
pred = {}
with open(predictions_file) as pred_file:
reader = csv.reader(pred_file, dialect=csv.excel_tab)
header = next(reader)
# If there are more than two columns, assume column 0 contains the ids, and
# columns 1-n contain class probabilities. Convert them to a class prediction
# using the specified `method`.
if len(header) > 2:
classes = [c for c in header[1:] if c]
if prediction_method is None:
prediction_method = "highest"
logger.info("No prediction method specified. Using 'highest'.")
if prediction_method == 'expected_value':
try:
classes = [int(c) for c in classes]
except ValueError as e:
raise e
for row in reader:
probabilities = [safe_float(p) for p in row[1:]]
prediction = get_prediction_from_probabilities(
classes, probabilities, prediction_method)
pred[row[0]] = safe_float(prediction)
else:
if prediction_method is not None:
logger.warning(
"A prediction method was provided, but the predictions "
"file doesn't contain probabilities. Ignoring prediction "
"method '{}'.".format(prediction_method))
for row in reader:
pred[row[0]] = safe_float(row[1])
# make a sorted list of example ids in order to match up
# labels and predictions
if set(gold.keys()) != set(pred.keys()):
raise ValueError('The example and prediction IDs do not match.')
example_ids = sorted(gold.keys())
res = {}
for metric_name in metric_names:
score = use_score_func(metric_name,
[gold[ex_id] for ex_id in example_ids],
[pred[ex_id] for ex_id in example_ids])
res[metric_name] = score
return res
def check_print_model_weights(task='classification'):
# create some simple classification or regression data
if task == 'classification' or task == 'classification_no_intercept':
train_fs, _ = make_classification_data(train_test_ratio=0.8)
elif task == 'multiclass_classification':
train_fs, _ = make_classification_data(train_test_ratio=0.8, num_labels=3)
else:
train_fs, _, _ = make_regression_data(num_features=4,
train_test_ratio=0.8)
# now train the appropriate model
if task == 'classification' or task == 'multiclass_classification':
learner = Learner('LogisticRegression')
learner.train(train_fs, grid_objective='f1_score_micro')
elif task == 'classification_no_intercept':
learner = Learner('LogisticRegression')
learner.train(train_fs, grid_objective='f1_score_micro', param_grid=[{'fit_intercept':[False]}])
elif task == 'regression':
learner = Learner('LinearRegression')
learner.train(train_fs, grid_objective='pearson')
else:
learner = Learner('LinearSVR')
learner.train(train_fs, grid_objective='pearson')
# now save the model to disk
model_file = join(_my_dir, 'output',
'test_print_model_weights.model')
learner.save(model_file)
# now call print_model_weights main() and capture the output
print_model_weights_cmd = [model_file]
err = ''
try:
old_stderr = sys.stderr
old_stdout = sys.stdout
sys.stderr = mystderr = StringIO()
sys.stdout = mystdout = StringIO()
pmw.main(print_model_weights_cmd)
out = mystdout.getvalue()
err = mystderr.getvalue()
finally:
sys.stderr = old_stderr
sys.stdout = old_stdout
print(err)
# now parse the output of the print_model_weight command
# and get the intercept and the feature values
if task == 'classification':
lines_to_parse = [l for l in out.split('\n')[1:] if l]
intercept = safe_float(lines_to_parse[0].split('\t')[0])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[2], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_[0])
assert_allclose(learner.model.coef_[0], feature_values)
elif task == 'multiclass_classification':
# for multiple classes we get an intercept for each class
# as well as a list of weights for each class
lines_to_parse = [l for l in out.split('\n')[1:] if l]
intercept = []
for intercept_string in lines_to_parse[0:3]:
intercept.append(safe_float(intercept_string.split('\t')[0]))
feature_values = [[], [], []]
for ltp in lines_to_parse[3:]:
fields = ltp.split('\t')
feature_values[int(fields[1])].append((fields[2], safe_float(fields[0])))
for index, weights in enumerate(feature_values):
feature_values[index] = [t[1] for t in sorted(weights)]
for index, weights in enumerate(learner.model.coef_):
assert_array_almost_equal(weights, feature_values[index])
assert_array_almost_equal(intercept, learner.model.intercept_)
elif task == 'classification_no_intercept':
lines_to_parse = [l for l in out.split('\n')[0:] if l]
intercept = safe_float(lines_to_parse[0].split('=')[1])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[2], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_[0], feature_values)
elif task == 'regression':
lines_to_parse = [l for l in out.split('\n') if l]
intercept = safe_float(lines_to_parse[0].split('=')[1])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[1], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_, feature_values)
else:
lines_to_parse = [l for l in out.split('\n') if l]
intercept_list = ast.literal_eval(lines_to_parse[0].split('=')[1].strip())
intercept = []
for intercept_string in intercept_list:
intercept.append(safe_float(intercept_string))
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[1], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_array_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_, feature_values)
def check_print_model_weights(task='classification'):
# create some simple classification or regression data
if task == 'classification':
train_fs, _ = make_classification_data(train_test_ratio=0.8)
elif task == 'multiclass_classification':
train_fs, _ = make_classification_data(train_test_ratio=0.8, num_labels=3)
else:
train_fs, _, _ = make_regression_data(num_features=4,
train_test_ratio=0.8)
# now train the appropriate model
if task == 'classification' or task == 'multiclass_classification':
learner = Learner('LogisticRegression')
learner.train(train_fs, grid_objective='f1_score_micro')
elif task == 'regression':
learner = Learner('LinearRegression')
learner.train(train_fs, grid_objective='pearson')
else:
learner = Learner('LinearSVR')
learner.train(train_fs, grid_objective='pearson')
# now save the model to disk
model_file = join(_my_dir, 'output',
'test_print_model_weights.model')
learner.save(model_file)
# now call print_model_weights main() and capture the output
print_model_weights_cmd = [model_file]
err = ''
try:
old_stderr = sys.stderr
old_stdout = sys.stdout
sys.stderr = mystderr = StringIO()
sys.stdout = mystdout = StringIO()
pmw.main(print_model_weights_cmd)
out = mystdout.getvalue()
err = mystderr.getvalue()
finally:
sys.stderr = old_stderr
sys.stdout = old_stdout
print(err)
# now parse the output of the print_model_weight command
# and get the intercept and the feature values
if task == 'classification':
lines_to_parse = [l for l in out.split('\n')[1:] if l]
intercept = safe_float(lines_to_parse[0].split('\t')[0])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[2], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_[0])
assert_allclose(learner.model.coef_[0], feature_values)
elif task == 'multiclass_classification':
# for multiple classes we get an intercept for each class
# as well as a list of weights for each class
lines_to_parse = [l for l in out.split('\n')[1:] if l]
intercept = []
for intercept_string in lines_to_parse[0:3]:
intercept.append(safe_float(intercept_string.split('\t')[0]))
feature_values = [[], [], []]
for ltp in lines_to_parse[3:]:
fields = ltp.split('\t')
feature_values[int(fields[1])].append((fields[2], safe_float(fields[0])))
for index, weights in enumerate(feature_values):
feature_values[index] = [t[1] for t in sorted(weights)]
for index, weights in enumerate(learner.model.coef_):
assert_array_almost_equal(weights, feature_values[index])
assert_array_almost_equal(intercept, learner.model.intercept_)
elif task == 'regression':
lines_to_parse = [l for l in out.split('\n') if l]
intercept = safe_float(lines_to_parse[0].split('=')[1])
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[1], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_, feature_values)
else:
lines_to_parse = [l for l in out.split('\n') if l]
intercept_list = ast.literal_eval(lines_to_parse[0].split('=')[1].strip())
intercept = []
for intercept_string in intercept_list:
intercept.append(safe_float(intercept_string))
feature_values = []
for ltp in lines_to_parse[1:]:
fields = ltp.split('\t')
feature_values.append((fields[1], safe_float(fields[0])))
feature_values = [t[1] for t in sorted(feature_values)]
assert_array_almost_equal(intercept, learner.model.intercept_)
assert_allclose(learner.model.coef_, feature_values)