def local_batch_predict(models, headers, test_reader, exclude, fields, resume,
output_path, max_models, number_of_tests, api, output,
verbosity, method, objective_field, session_file,
debug):
"""Get local predictions form partial Multimodel, combine and save to file
"""
def draw_progress_bar(current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Predicted on %s out of %s models [%s%%]" % (
localize(current), localize(total), pct))
models_total = len(models)
models_splits = [models[index:(index + max_models)] for index
in range(0, models_total, max_models)]
input_data_list = []
for row in test_reader:
for index in exclude:
del row[index]
input_data_list.append(fields.pair(row, headers,
objective_field))
total_votes = []
models_count = 0
for models_split in models_splits:
if resume:
for model in models_split:
pred_file = get_predictions_file_name(model,
output_path)
u.checkpoint(u.are_predictions_created,
pred_file,
number_of_tests, debug=debug)
complete_models = []
for index in range(len(models_split)):
complete_models.append(api.check_resource(
models_split[index], api.get_model))
local_model = MultiModel(complete_models)
local_model.batch_predict(input_data_list,
output_path, reuse=True)
votes = local_model.batch_votes(output_path)
models_count += max_models
if models_count > models_total:
models_count = models_total
if verbosity:
draw_progress_bar(models_count, models_total)
if total_votes:
for index in range(0, len(votes)):
predictions = total_votes[index].predictions
predictions.extend(votes[index].predictions)
else:
total_votes = votes
message = u.dated("Combining predictions.\n")
u.log_message(message, log_file=session_file, console=verbosity)
for multivote in total_votes:
u.write_prediction(multivote.combine(method), output)
def remote_predict(models, test_reader, prediction_file, api,
resume=False,
verbosity=True, output_path=None,
method=PLURALITY_CODE, tags="",
session_file=None, log=None, debug=False):
"""Retrieve predictions remotely, combine them and save predictions to file
"""
predictions_files = []
prediction_args = {
"tags": tags
}
test_set_header = test_reader.has_headers()
if output_path is None:
output_path = u.check_dir(prediction_file)
message_logged = False
for model in models:
model = bigml.api.get_model_id(model)
predictions_file = get_predictions_file_name(model,
output_path)
predictions_files.append(predictions_file)
if (not resume or
not u.checkpoint(u.are_predictions_created, predictions_file,
test_reader.number_of_tests(), debug=debug)):
if not message_logged:
message = u.dated("Creating remote predictions.")
u.log_message(message, log_file=session_file,
console=verbosity)
message_logged = True
predictions_file = csv.writer(open(predictions_file, 'w', 0))
for input_data in test_reader:
prediction = api.create_prediction(model, input_data,
by_name=test_set_header,
wait_time=0,
args=prediction_args)
u.check_resource_error(prediction,
"Failed to create prediction: ")
u.log_message("%s\n" % prediction['resource'], log_file=log)
prediction_row = u.prediction_to_row(prediction)
predictions_file.writerow(prediction_row)
u.combine_votes(predictions_files,
Model(models[0]).to_prediction,
prediction_file, method)
def remote_predict(models, headers, output_path, number_of_tests, resume,
verbosity, test_reader, exclude, fields, api,
prediction_file, method, tags, objective_field,
session_file, test_set_header, log, debug):
"""Retrieve predictions remotely, combine them and save predictions to file
"""
predictions_files = []
prediction_args = {
"tags": tags
}
for model in models:
if not isinstance(model, basestring) and 'resource' in model:
model = model['resource']
predictions_file = get_predictions_file_name(model,
output_path)
predictions_files.append(predictions_file)
if (not resume or
not u.checkpoint(u.are_predictions_created, predictions_file,
number_of_tests, debug=debug)):
message = u.dated("Creating remote predictions.\n")
u.log_message(message, log_file=session_file,
console=verbosity)
predictions_file = csv.writer(open(predictions_file, 'w', 0))
for row in test_reader:
for index in exclude:
del row[index]
input_data = fields.pair(row, headers, objective_field)
prediction = api.create_prediction(model, input_data,
by_name=test_set_header,
wait_time=0,
args=prediction_args)
u.log_message("%s\n" % prediction['resource'], log_file=log)
prediction_row = u.prediction_to_row(prediction)
predictions_file.writerow(prediction_row)
u.combine_votes(predictions_files,
Model(models[0]).to_prediction,
prediction_file, method)
def compute_output(api, args, training_set, test_set=None, output=None,
objective_field=None,
description=None,
field_attributes=None,
types=None,
dataset_fields=None,
model_fields=None,
name=None, training_set_header=True,
test_set_header=True, model_ids=None,
votes_files=None, resume=False, fields_map=None):
""" Creates one or more models using the `training_set` or uses the ids
of previously created BigML models to make predictions for the `test_set`.
"""
source = None
dataset = None
model = None
models = None
fields = None
path = u.check_dir(output)
session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
csv_properties = {}
# If logging is required, open the file for logging
log = None
if args.log_file:
u.check_dir(args.log_file)
log = args.log_file
# If --clear_logs the log files are cleared
if args.clear_logs:
try:
open(log, 'w', 0).close()
except IOError:
pass
if (training_set or (args.evaluate and test_set)):
if resume:
resume, args.source = u.checkpoint(u.is_source_created, path,
bigml.api, debug=args.debug)
if not resume:
message = u.dated("Source not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set = None
if (training_set and not args.source and not args.dataset and
not args.model and not args.models):
data_set = training_set
data_set_header = training_set_header
elif (args.evaluate and test_set and not args.source):
data_set = test_set
data_set_header = test_set_header
if not data_set is None:
source_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag,
"source_parser": {"header": data_set_header}}
message = u.dated("Creating source.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
source = api.create_source(data_set, source_args,
progress_bar=args.progress_bar)
source = api.check_resource(source, api.get_source)
message = u.dated("Source created: %s\n" % u.get_url(source, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
u.log_message("%s\n" % source['resource'], log_file=log)
fields = Fields(source['object']['fields'],
source['object']['source_parser']['missing_tokens'],
source['object']['source_parser']['locale'])
source_file = open(path + '/source', 'w', 0)
source_file.write("%s\n" % source['resource'])
source_file.write("%s\n" % source['object']['name'])
source_file.flush()
source_file.close()
# If a source is provided, we retrieve it.
elif args.source:
message = u.dated("Retrieving source. %s\n" %
u.get_url(args.source, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
source = api.get_source(args.source)
# If we already have source, we check that is finished and extract the
# fields, and update them if needed.
if source:
if source['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.check_resource(source, api.get_source)
csv_properties = {'missing_tokens':
source['object']['source_parser']['missing_tokens'],
'data_locale':
source['object']['source_parser']['locale']}
fields = Fields(source['object']['fields'], **csv_properties)
update_fields = {}
if field_attributes:
for (column, value) in field_attributes.iteritems():
update_fields.update({
fields.field_id(column): value})
message = u.dated("Updating source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.update_source(source, {"fields": update_fields})
update_fields = {}
if types:
for (column, value) in types.iteritems():
update_fields.update({
fields.field_id(column): {'optype': value}})
message = u.dated("Updating source. %s\n" %
u.get_url(source, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
source = api.update_source(source, {"fields": update_fields})
if (training_set or args.source or (args.evaluate and test_set)):
if resume:
resume, args.dataset = u.checkpoint(u.is_dataset_created, path,
bigml.api,
debug=args.debug)
if not resume:
message = u.dated("Dataset not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If we have a source but not dataset or model has been provided, we
# create a new dataset if the no_dataset option isn't set up. Also
# if evaluate is set and test_set has been provided.
if ((source and not args.dataset and not args.model and not model_ids and
not args.no_dataset) or
(args.evaluate and args.test_set and not args.dataset)):
dataset_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag
}
if args.json_filter:
dataset_args.update(json_filter=args.json_filter)
elif args.lisp_filter:
dataset_args.update(lisp_filter=args.lisp_filter)
input_fields = []
if dataset_fields:
for name in dataset_fields:
input_fields.append(fields.field_id(name))
dataset_args.update(input_fields=input_fields)
message = u.dated("Creating dataset.\n")
u.log_message(message, log_file=session_file, console=args.verbosity)
dataset = api.create_dataset(source, dataset_args)
dataset = api.check_resource(dataset, api.get_dataset)
message = u.dated("Dataset created: %s\n" % u.get_url(dataset, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
u.log_message("%s\n" % dataset['resource'], log_file=log)
dataset_file = open(path + '/dataset', 'w', 0)
dataset_file.write("%s\n" % dataset['resource'])
dataset_file.flush()
dataset_file.close()
# If a dataset is provided, let's retrieve it.
elif args.dataset:
message = u.dated("Retrieving dataset. %s\n" %
u.get_url(args.dataset, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
dataset = api.get_dataset(args.dataset)
# If we already have a dataset, we check the status and get the fields if
# we hadn't them yet.
if dataset:
if dataset['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
dataset = api.check_resource(dataset, api.get_dataset)
if not csv_properties:
csv_properties = {'data_locale':
dataset['object']['locale']}
if args.public_dataset:
if not description:
raise Exception("You should provide a description to publish.")
public_dataset = {"private": False}
if args.dataset_price:
message = u.dated("Updating dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_dataset.update(price=args.dataset_price)
message = u.dated("Updating dataset. %s\n" %
u.get_url(dataset, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
dataset = api.update_dataset(dataset, public_dataset)
fields = Fields(dataset['object']['fields'], **csv_properties)
# If we have a dataset but not a model, we create the model if the no_model
# flag hasn't been set up.
if (dataset and not args.model and not model_ids and not args.no_model):
model_args = {
"name": name,
"description": description,
"category": args.category,
"tags": args.tag
}
if objective_field is not None:
model_args.update({"objective_field":
fields.field_id(objective_field)})
# If evaluate flag is on, we choose a deterministic sampling with 80%
# of the data to create the model
if args.evaluate:
if args.sample_rate == 1:
args.sample_rate = EVALUATE_SAMPLE_RATE
seed = SEED
model_args.update(seed=seed)
input_fields = []
if model_fields:
for name in model_fields:
input_fields.append(fields.field_id(name))
model_args.update(input_fields=input_fields)
if args.pruning and args.pruning != 'smart':
model_args.update(stat_pruning=(args.pruning == 'statistical'))
model_args.update(sample_rate=args.sample_rate,
replacement=args.replacement,
randomize=args.randomize)
model_ids = []
models = []
if resume:
resume, model_ids = u.checkpoint(u.are_models_created, path,
args.number_of_models,
bigml.api, debug=args.debug)
if not resume:
message = u.dated("Found %s models out of %s. Resuming.\n" %
(len(model_ids),
args.number_of_models))
u.log_message(message, log_file=session_file,
console=args.verbosity)
models = model_ids
args.number_of_models -= len(model_ids)
model_file = open(path + '/models', 'w', 0)
for model_id in model_ids:
model_file.write("%s\n" % model_id)
last_model = None
if args.number_of_models > 0:
message = u.dated("Creating %s.\n" %
u.plural("model", args.number_of_models))
u.log_message(message, log_file=session_file,
console=args.verbosity)
for i in range(1, args.number_of_models + 1):
if i > args.max_parallel_models:
api.check_resource(last_model, api.get_model)
model = api.create_model(dataset, model_args)
u.log_message("%s\n" % model['resource'], log_file=log)
last_model = model
model_ids.append(model['resource'])
models.append(model)
model_file.write("%s\n" % model['resource'])
model_file.flush()
if args.number_of_models < 2 and args.verbosity:
if model['object']['status']['code'] != bigml.api.FINISHED:
model = api.check_resource(model, api.get_model)
models[0] = model
message = u.dated("Model created: %s.\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
model_file.close()
# If a model is provided, we retrieve it.
elif args.model:
message = u.dated("Retrieving model. %s\n" %
u.get_url(args.model, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
model = api.get_model(args.model)
elif args.models or args.model_tag:
models = model_ids[:]
if model_ids and test_set and not args.evaluate:
model_id = ""
if len(model_ids) == 1:
model_id = model_ids[0]
message = u.dated("Retrieving %s. %s\n" %
(u.plural("model", len(model_ids)),
u.get_url(model_id, api)))
u.log_message(message, log_file=session_file, console=args.verbosity)
if len(model_ids) < args.max_batch_models:
models = []
for model in model_ids:
model = api.check_resource(model, api.get_model)
models.append(model)
model = models[0]
else:
model = api.check_resource(model_ids[0], api.get_model)
models[0] = model
# We check that the model is finished and get the fields if haven't got
# them yet.
if model and not args.evaluate and (test_set or args.black_box
or args.white_box):
if model['object']['status']['code'] != bigml.api.FINISHED:
message = u.dated("Retrieving model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
model = api.check_resource(model, api.get_model)
if args.black_box:
if not description:
raise Exception("You should provide a description to publish.")
model = api.update_model(model, {"private": False})
if args.white_box:
if not description:
raise Exception("You should provide a description to publish.")
public_model = {"private": False, "white_box": True}
if args.model_price:
message = u.dated("Updating model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_model.update(price=args.model_price)
if args.cpp:
message = u.dated("Updating model. %s\n" %
u.get_url(model, api))
u.log_message(message, log_file=session_file,
console=args.verbosity)
public_model.update(credits_per_prediction=args.cpp)
model = api.update_model(model, public_model)
if not csv_properties:
csv_properties = {'data_locale':
model['object']['locale']}
csv_properties.update(verbose=True)
if args.user_locale:
csv_properties.update(data_locale=args.user_locale)
fields = Fields(model['object']['model']['fields'], **csv_properties)
if model and not models:
models = [model]
if models and test_set and not args.evaluate:
objective_field = models[0]['object']['objective_fields']
if isinstance(objective_field, list):
objective_field = objective_field[0]
predict(test_set, test_set_header, models, fields, output,
objective_field, args.remote, api, log,
args.max_batch_models, args.method, resume, args.tag,
args.verbosity, session_file, args.debug)
# When combine_votes flag is used, retrieve the predictions files saved
# in the comma separated list of directories and combine them
if votes_files:
model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$',
r'\1', votes_files[0]).replace("_", "/")
model = api.check_resource(model_id, api.get_model)
local_model = Model(model)
message = u.dated("Combining votes.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
u.combine_votes(votes_files, local_model.to_prediction,
output, args.method)
# If evaluate flag is on, create remote evaluation and save results in
# json and human-readable format.
if args.evaluate:
if resume:
resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
bigml.api,
debug=args.debug)
if not resume:
message = u.dated("Evaluation not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_file = open(path + '/evaluation', 'w', 0)
evaluation_args = {
"name": name,
"description": description,
"tags": args.tag
}
if not fields_map is None:
update_map = {}
for (dataset_column, model_column) in fields_map.iteritems():
update_map.update({
fields.field_id(dataset_column):
fields.field_id(model_column)})
evaluation_args.update({"fields_map": update_map})
if not ((args.dataset or args.test_set)
and (args.model or args.models or args.model_tag)):
evaluation_args.update(out_of_bag=True, seed=SEED,
sample_rate=args.sample_rate)
message = u.dated("Creating evaluation.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
evaluation = api.create_evaluation(model, dataset, evaluation_args)
u.log_message("%s\n" % evaluation['resource'], log_file=log)
evaluation_file.write("%s\n" % evaluation['resource'])
evaluation_file.flush()
evaluation_file.close()
message = u.dated("Retrieving evaluation. %s\n" %
u.get_url(evaluation, api))
u.log_message(message, log_file=session_file, console=args.verbosity)
evaluation = api.check_resource(evaluation, api.get_evaluation)
evaluation_json = open(output + '.json', 'w', 0)
evaluation_json.write(json.dumps(evaluation['object']['result']))
evaluation_json.flush()
evaluation_json.close()
evaluation_txt = open(output + '.txt', 'w', 0)
api.pprint(evaluation['object']['result'],
evaluation_txt)
evaluation_txt.flush()
evaluation_txt.close()
# Workaround to restore windows console cp850 encoding to print the tree
if sys.platform == "win32" and sys.stdout.isatty():
import locale
data_locale = locale.getlocale()
if not data_locale[0] is None:
locale.setlocale(locale.LC_ALL, (data_locale[0], "850"))
message = (u"\nGenerated files:\n\n" +
unicode(u.print_tree(path, " "), "utf-8") + u"\n")
else:
message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
u.log_message(message, log_file=session_file, console=args.verbosity)
def local_batch_predict(models, test_reader, prediction_file, api,
max_models=MAX_MODELS,
resume=False, output_path=None, output=None,
verbosity=True, method=PLURALITY_CODE,
session_file=None, debug=False):
"""Get local predictions form partial Multimodel, combine and save to file
"""
def draw_progress_bar(current, total):
"""Draws a text based progress report.
"""
pct = 100 - ((total - current) * 100) / (total)
console_log("Predicted on %s out of %s models [%s%%]" % (
localize(current), localize(total), pct))
test_set_header = test_reader.has_headers()
if output_path is None:
output_path = u.check_dir(prediction_file)
if output is None:
try:
output = open(prediction_file, 'w', 0)
except IOError:
raise IOError("Failed to write in %s" % prediction_file)
models_total = len(models)
models_splits = [models[index:(index + max_models)] for index
in range(0, models_total, max_models)]
input_data_list = []
for input_data in test_reader:
input_data_list.append(input_data)
total_votes = []
models_count = 0
for models_split in models_splits:
if resume:
for model in models_split:
pred_file = get_predictions_file_name(model,
output_path)
u.checkpoint(u.are_predictions_created,
pred_file,
test_reader.number_of_tests(), debug=debug)
complete_models = []
for index in range(len(models_split)):
model = models_split[index]
if (isinstance(model, basestring) or
bigml.api.get_status(model)['code'] != bigml.api.FINISHED):
try:
model = api.check_resource(model, api.get_model,
'limit=-1')
except ValueError, exception:
sys.exit("Failed to get model: %s" % (model,
str(exception)))
complete_models.append(model)
local_model = MultiModel(complete_models)
local_model.batch_predict(input_data_list,
output_path,
by_name=test_set_header,
reuse=True)
votes = local_model.batch_votes(output_path)
models_count += max_models
if models_count > models_total:
models_count = models_total
if verbosity:
draw_progress_bar(models_count, models_total)
if total_votes:
for index in range(0, len(votes)):
predictions = total_votes[index].predictions
predictions.extend(votes[index].predictions)
else:
total_votes = votes
def compute_output(api, args, training_set, test_set=None, output=None,
objective_field=None,
description=None,
field_attributes=None,
types=None,
dataset_fields=None,
model_fields=None,
name=None, training_set_header=True,
test_set_header=True, model_ids=None,
votes_files=None, resume=False, fields_map=None):
""" Creates one or more models using the `training_set` or uses the ids
of previously created BigML models to make predictions for the `test_set`.
"""
source = None
dataset = None
model = None
models = None
fields = None
# It is compulsory to have a description to publish either datasets or
# models
if (not description and
(args.black_box or args.white_box or args.public_dataset)):
raise Exception("You should provide a description to publish.")
path = u.check_dir(output)
session_file = "%s%s%s" % (path, os.sep, SESSIONS_LOG)
csv_properties = {}
# If logging is required, open the file for logging
log = None
if args.log_file:
u.check_dir(args.log_file)
log = args.log_file
# If --clear_logs the log files are cleared
if args.clear_logs:
try:
open(log, 'w', 0).close()
except IOError:
pass
# Starting source processing
if (training_set or (args.evaluate and test_set)):
# If resuming, try to extract args.source form log files
if resume:
resume, args.source = u.checkpoint(u.is_source_created, path,
debug=args.debug)
if not resume:
message = u.dated("Source not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If neither a previous source, dataset or model are provided.
# we create a new one. Also if --evaluate and test data are provided
# we create a new dataset to test with.
data_set, data_set_header = r.data_to_source(training_set, test_set,
training_set_header,
test_set_header, args)
if data_set is not None:
source_args = r.set_source_args(data_set_header, name, description,
args)
source = r.create_source(data_set, source_args, args, api,
path, session_file, log)
# If a source is provided either through the command line or in resume
# steps, we use it.
elif args.source:
source = bigml.api.get_source_id(args.source)
# If we already have source, we check that is finished, extract the
# fields, and update them if needed.
if source:
source = r.get_source(source, api, args.verbosity, session_file)
if 'source_parser' in source['object']:
source_parser = source['object']['source_parser']
if 'missing_tokens' in source_parser:
csv_properties['missing_tokens'] = (
source_parser['missing_tokens'])
if 'data_locale' in source_parser:
csv_properties['data_locale'] = source_parser['locale']
fields = Fields(source['object']['fields'], **csv_properties)
if field_attributes:
source = r.update_source_fields(source, field_attributes, fields,
api, args.verbosity,
session_file)
if types:
source = r.update_source_fields(source, types, fields, api,
args.verbosity, session_file)
# End of source processing
# Starting dataset processing
if (training_set or args.source or (args.evaluate and test_set)):
# if resuming, try to extract args.dataset form log files
if resume:
resume, args.dataset = u.checkpoint(u.is_dataset_created, path,
debug=args.debug)
if not resume:
message = u.dated("Dataset not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
# If we have a source but not dataset or model has been provided, we
# create a new dataset if the no_dataset option isn't set up. Also
# if evaluate is set and test_set has been provided.
if ((source and not args.dataset and not args.model and not model_ids and
not args.no_dataset) or
(args.evaluate and args.test_set and not args.dataset)):
dataset_args = r.set_dataset_args(name, description, args, fields,
dataset_fields)
dataset = r.create_dataset(source, dataset_args, args, api,
path, session_file, log)
# If a dataset is provided, let's retrieve it.
elif args.dataset:
dataset = bigml.api.get_dataset_id(args.dataset)
# If we already have a dataset, we check the status and get the fields if
# we hadn't them yet.
if dataset:
dataset = r.get_dataset(dataset, api, args.verbosity, session_file)
if not csv_properties and 'locale' in dataset['object']:
csv_properties = {
'data_locale': dataset['object']['locale']}
fields = Fields(dataset['object']['fields'], **csv_properties)
if args.public_dataset:
r.publish_dataset(dataset, api, args, session_file)
#end of dataset processing
#start of model processing
# If we have a dataset but not a model, we create the model if the no_model
# flag hasn't been set up.
if (dataset and not args.model and not model_ids and not args.no_model):
model_ids = []
models = []
if resume:
resume, model_ids = u.checkpoint(u.are_models_created, path,
args.number_of_models,
debug=args.debug)
if not resume:
message = u.dated("Found %s models out of %s. Resuming.\n" %
(len(model_ids),
args.number_of_models))
u.log_message(message, log_file=session_file,
console=args.verbosity)
models = model_ids
args.number_of_models -= len(model_ids)
model_args = r.set_model_args(name, description, args,
objective_field, fields, model_fields)
models, model_ids = r.create_models(dataset, models,
model_args, args, api,
path, session_file, log)
model = models[0]
# If a model is provided, we use it.
elif args.model:
model = args.model
model_ids = [model]
models = [model]
elif args.models or args.model_tag:
models = model_ids[:]
model = models[0]
# If we are going to predict we must retrieve the models
if model_ids and test_set and not args.evaluate:
models, model_ids = r.get_models(models, args, api, session_file)
model = models[0]
# We get the fields of the model if we haven't got
# them yet and update its public state if needed
if model and not args.evaluate and (test_set or args.black_box
or args.white_box):
if args.black_box or args.white_box:
model = r.publish_model(model, args, api, session_file)
models[0] = model
if not csv_properties:
csv_properties = {}
csv_properties.update(verbose=True)
if args.user_locale is None:
args.user_locale = model['object'].get('locale', None)
csv_properties.update(data_locale=args.user_locale)
if 'model_fields' in model['object']['model']:
model_fields = model['object']['model']['model_fields'].keys()
csv_properties.update(include=model_fields)
if 'missing_tokens' in model['object']['model']:
missing_tokens = model['object']['model']['missing_tokens']
else:
missing_tokens = MISSING_TOKENS
csv_properties.update(missing_tokens=missing_tokens)
objective_field = models[0]['object']['objective_fields']
if isinstance(objective_field, list):
objective_field = objective_field[0]
csv_properties.update(objective_field=objective_field)
fields = Fields(model['object']['model']['fields'], **csv_properties)
# end of model processing
# If predicting
if models and test_set and not args.evaluate:
predict(test_set, test_set_header, models, fields, output,
objective_field, args.remote, api, log,
args.max_batch_models, args.method, resume, args.tag,
args.verbosity, session_file, args.debug)
# When combine_votes flag is used, retrieve the predictions files saved
# in the comma separated list of directories and combine them
if votes_files:
model_id = re.sub(r'.*(model_[a-f0-9]{24})__predictions\.csv$',
r'\1', votes_files[0]).replace("_", "/")
try:
model = api.check_resource(model_id, api.get_model)
except ValueError, exception:
sys.exit("Failed to get model %s: %s" % (model_id, str(exception)))
local_model = Model(model)
message = u.dated("Combining votes.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
u.combine_votes(votes_files, local_model.to_prediction,
output, args.method)
model = api.check_resource(model_id, api.get_model)
except ValueError, exception:
sys.exit("Failed to get model %s: %s" % (model_id, str(exception)))
local_model = Model(model)
message = u.dated("Combining votes.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
u.combine_votes(votes_files, local_model.to_prediction,
output, args.method)
# If evaluate flag is on, create remote evaluation and save results in
# json and human-readable format.
if args.evaluate:
if resume:
resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
debug=args.debug)
if not resume:
message = u.dated("Evaluation not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
evaluation = r.create_evaluation(model, dataset, evaluation_args,
args, api, path, session_file,
log)
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
r.save_evaluation(evaluation, output, api)
