def local_batch_predict(models, headers, test_reader, exclude, fields, resume, output_path, max_models, number_of_tests, api, output, verbosity, method, objective_field, session_file, debug): """Get local predictions form partial Multimodel, combine and save to file """ def draw_progress_bar(current, total): """Draws a text based progress report. """ pct = 100 - ((total - current) * 100) / (total) console_log("Predicted on %s out of %s models [%s%%]" % ( localize(current), localize(total), pct)) models_total = len(models) models_splits = [models[index:(index + max_models)] for index in range(0, models_total, max_models)] input_data_list = [] for row in test_reader: for index in exclude: del row[index] input_data_list.append(fields.pair(row, headers, objective_field)) total_votes = [] models_count = 0 for models_split in models_splits: if resume: for model in models_split: pred_file = get_predictions_file_name(model, output_path) u.checkpoint(u.are_predictions_created, pred_file, number_of_tests, debug=debug) complete_models = [] for index in range(len(models_split)): complete_models.append(api.check_resource( models_split[index], api.get_model)) local_model = MultiModel(complete_models) local_model.batch_predict(input_data_list, output_path, reuse=True) votes = local_model.batch_votes(output_path) models_count += max_models if models_count > models_total: models_count = models_total if verbosity: draw_progress_bar(models_count, models_total) if total_votes: for index in range(0, len(votes)): predictions = total_votes[index].predictions predictions.extend(votes[index].predictions) else: total_votes = votes message = u.dated("Combining predictions.\n") u.log_message(message, log_file=session_file, console=verbosity) for multivote in total_votes: u.write_prediction(multivote.combine(method), output)
def local_predict(models, test_reader, output, method, prediction_info=None): """Get local predictions and combine them to get a final prediction """ local_model = MultiModel(models) test_set_header = test_reader.has_headers() for input_data in test_reader: prediction = local_model.predict(input_data, by_name=test_set_header, method=method, with_confidence=True) u.write_prediction(prediction, output, prediction_info)
def local_predict(models, test_reader, output, method): """Get local predictions and combine them to get a final prediction """ local_model = MultiModel(models) test_set_header = test_reader.has_headers() for input_data in test_reader: prediction = local_model.predict(input_data, by_name=test_set_header, method=method, with_confidence=True) u.write_prediction(prediction, output)
def local_predict(models, headers, test_reader, exclude, fields, method, objective_field, output, test_set_header): """Get local predictions, combine them and save predictions to file """ local_model = MultiModel(models) for row in test_reader: for index in exclude: del row[index] input_data = fields.pair(row, headers, objective_field) prediction = local_model.predict(input_data, by_name=test_set_header, method=method) u.write_prediction(prediction, output)
votes = local_model.batch_votes(output_path) models_count += max_models if models_count > models_total: models_count = models_total if verbosity: draw_progress_bar(models_count, models_total) if total_votes: for index in range(0, len(votes)): predictions = total_votes[index].predictions predictions.extend(votes[index].predictions) else: total_votes = votes message = u.dated("Combining predictions.\n") u.log_message(message, log_file=session_file, console=verbosity) for multivote in total_votes: u.write_prediction(multivote.combine(method, True), output, prediction_info) def predict(test_set, test_set_header, models, fields, output, objective_field, remote=False, api=None, log=None, max_models=MAX_MODELS, method=0, resume=False, tags=None,
votes = local_model.batch_votes(output_path) models_count += max_models if models_count > models_total: models_count = models_total if verbosity: draw_progress_bar(models_count, models_total) if total_votes: for index in range(0, len(votes)): predictions = total_votes[index].predictions predictions.extend(votes[index].predictions) else: total_votes = votes message = u.dated("Combining predictions.\n") u.log_message(message, log_file=session_file, console=verbosity) for multivote in total_votes: u.write_prediction(multivote.combine(method, True), output) def predict(test_set, test_set_header, models, fields, output, objective_field, remote=False, api=None, log=None, max_models=MAX_MODELS, method=0, resume=False, tags=None, verbosity=1, session_file=None, debug=False): """Computes a prediction for each entry in the `test_set`. Predictions can be computed remotely, locally using MultiModels built on all the models or locally using MultiModels on subgroups of models. Chosing a max_batch_models value not bigger than the number_of_models flag will lead to the last case, where memory usage is bounded and each model predictions are saved for further use. """