def local_batch_predict(models, headers, test_reader, exclude, fields, resume,
output_path, max_models, number_of_tests, api, output,
verbosity, method, objective_field, session_file,
debug):
"""Get local predictions form partial Multimodel, combine and save to file
"""
def draw_progress_bar(current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Predicted on %s out of %s models [%s%%]" % (
localize(current), localize(total), pct))
models_total = len(models)
models_splits = [models[index:(index + max_models)] for index
in range(0, models_total, max_models)]
input_data_list = []
for row in test_reader:
for index in exclude:
del row[index]
input_data_list.append(fields.pair(row, headers,
objective_field))
total_votes = []
models_count = 0
for models_split in models_splits:
if resume:
for model in models_split:
pred_file = get_predictions_file_name(model,
output_path)
u.checkpoint(u.are_predictions_created,
pred_file,
number_of_tests, debug=debug)
complete_models = []
for index in range(len(models_split)):
complete_models.append(api.check_resource(
models_split[index], api.get_model))
local_model = MultiModel(complete_models)
local_model.batch_predict(input_data_list,
output_path, reuse=True)
votes = local_model.batch_votes(output_path)
models_count += max_models
if models_count > models_total:
models_count = models_total
if verbosity:
draw_progress_bar(models_count, models_total)
if total_votes:
for index in range(0, len(votes)):
predictions = total_votes[index].predictions
predictions.extend(votes[index].predictions)
else:
total_votes = votes
message = u.dated("Combining predictions.\n")
u.log_message(message, log_file=session_file, console=verbosity)
for multivote in total_votes:
u.write_prediction(multivote.combine(method), output)
def local_predict(models, test_reader, output, method, prediction_info=None):
"""Get local predictions and combine them to get a final prediction
"""
local_model = MultiModel(models)
test_set_header = test_reader.has_headers()
for input_data in test_reader:
prediction = local_model.predict(input_data,
by_name=test_set_header,
method=method,
with_confidence=True)
u.write_prediction(prediction, output, prediction_info)
def local_predict(models, test_reader, output, method):
"""Get local predictions and combine them to get a final prediction
"""
local_model = MultiModel(models)
test_set_header = test_reader.has_headers()
for input_data in test_reader:
prediction = local_model.predict(input_data,
by_name=test_set_header,
method=method,
with_confidence=True)
u.write_prediction(prediction, output)
def local_predict(models, headers, test_reader, exclude, fields, method,
objective_field, output, test_set_header):
"""Get local predictions, combine them and save predictions to file
"""
local_model = MultiModel(models)
for row in test_reader:
for index in exclude:
del row[index]
input_data = fields.pair(row, headers, objective_field)
prediction = local_model.predict(input_data,
by_name=test_set_header,
method=method)
u.write_prediction(prediction, output)
votes = local_model.batch_votes(output_path)
models_count += max_models
if models_count > models_total:
models_count = models_total
if verbosity:
draw_progress_bar(models_count, models_total)
if total_votes:
for index in range(0, len(votes)):
predictions = total_votes[index].predictions
predictions.extend(votes[index].predictions)
else:
total_votes = votes
message = u.dated("Combining predictions.\n")
u.log_message(message, log_file=session_file, console=verbosity)
for multivote in total_votes:
u.write_prediction(multivote.combine(method, True), output,
prediction_info)
def predict(test_set,
test_set_header,
models,
fields,
output,
objective_field,
remote=False,
api=None,
log=None,
max_models=MAX_MODELS,
method=0,
resume=False,
tags=None,
votes = local_model.batch_votes(output_path)
models_count += max_models
if models_count > models_total:
models_count = models_total
if verbosity:
draw_progress_bar(models_count, models_total)
if total_votes:
for index in range(0, len(votes)):
predictions = total_votes[index].predictions
predictions.extend(votes[index].predictions)
else:
total_votes = votes
message = u.dated("Combining predictions.\n")
u.log_message(message, log_file=session_file, console=verbosity)
for multivote in total_votes:
u.write_prediction(multivote.combine(method, True), output)
def predict(test_set, test_set_header, models, fields, output,
objective_field, remote=False, api=None, log=None,
max_models=MAX_MODELS, method=0, resume=False,
tags=None, verbosity=1, session_file=None, debug=False):
"""Computes a prediction for each entry in the `test_set`.
Predictions can be computed remotely, locally using MultiModels built
on all the models or locally using MultiModels on subgroups of models.
Chosing a max_batch_models value not bigger than the number_of_models
flag will lead to the last case, where memory usage is bounded and each
model predictions are saved for further use.
"""
