def evaluate(time_series_set, datasets, api, args, resume,
session_file=None, path=None, log=None,
fields=None, dataset_fields=None,
objective_field=None):
"""Evaluates a list of time-series with the given dataset
"""
output = args.predictions
evaluation_files = []
evaluations, resume = evaluations_process(
time_series_set, datasets, fields,
dataset_fields, api, args, resume,
session_file=session_file, path=path, log=log,
objective_field=objective_field)
for index in range(0, len(evaluations)):
evaluation = evaluations[index]
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
if r.shared_changed(args.shared, evaluation):
evaluation_args = {"shared": args.shared}
evaluation = r.update_evaluation(evaluation, evaluation_args,
args, api=api, path=path,
session_file=session_file)
file_name = output
r.save_evaluation(evaluation, file_name, api)
return resume
def cross_validate(models,
dataset,
fields,
api,
args,
resume,
session_file=None,
path=None,
log=None):
"""Cross-validates using a MONTE-CARLO variant
"""
evaluations, resume = evaluations_process(models, [dataset],
fields,
fields,
api,
args,
resume,
session_file=session_file,
path=path,
log=log)
if not resume:
evaluation_files = []
for evaluation in evaluations:
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
model_id = evaluation['object']['model']
file_name = "%s%s%s__evaluation" % (path, os.sep,
model_id.replace("/", "_"))
evaluation_files.append(file_name + ".json")
r.save_evaluation(evaluation, file_name, api)
cross_validation = average_evaluations(evaluation_files)
file_name = "%s%scross_validation" % (path, os.sep)
r.save_evaluation(cross_validation, file_name, api)
def evaluate(model, dataset, name, description, fields, fields_map, output,
api, args, resume,
session_file=None, path=None, log=None):
"""Evaluates a model or an ensemble with the given dataset
"""
if resume:
message = u.dated("Evaluation not found. Resuming.\n")
resume, evaluation = c.checkpoint(
c.is_evaluation_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
if args.ensemble:
model_or_ensemble = args.ensemble
else:
model_or_ensemble = model
evaluation = r.create_evaluation(model_or_ensemble, dataset,
evaluation_args,
args, api, path, session_file,
log)
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
r.save_evaluation(evaluation, output, api)
return resume
def evaluate(models_or_ensembles, datasets, output, api, args, resume,
session_file=None, path=None, log=None, name=None,
description=None, fields=None, dataset_fields=None,
fields_map=None, labels=None, all_labels=None,
objective_field=None):
"""Evaluates a list of models or ensembles with the given dataset
"""
evaluation_files = []
evaluations, resume = evaluations_process(
models_or_ensembles, datasets, name, description, fields,
dataset_fields, fields_map, api, args, resume,
session_file=session_file, path=path, log=log,
labels=labels, all_labels=all_labels, objective_field=objective_field)
if args.multi_label:
file_labels = map(slugify,
u.objective_field_names(models_or_ensembles, api))
for index in range(0, len(evaluations)):
evaluation = evaluations[index]
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
file_name = output
if args.multi_label:
suffix = file_labels[index]
file_name += "_%s" % suffix
evaluation_files.append("%s.json" % file_name)
r.save_evaluation(evaluation, file_name, api)
if args.multi_label:
mean_evaluation = average_evaluations(evaluation_files)
r.save_evaluation(mean_evaluation, output, api)
return resume
def evaluate(models_or_ensembles,
datasets,
api,
args,
resume,
session_file=None,
path=None,
log=None,
fields=None,
dataset_fields=None,
labels=None,
all_labels=None,
objective_field=None):
"""Evaluates a list of models or ensembles with the given dataset
"""
output = args.predictions
evaluation_files = []
evaluations, resume = evaluations_process(models_or_ensembles,
datasets,
fields,
dataset_fields,
api,
args,
resume,
session_file=session_file,
path=path,
log=log,
labels=labels,
all_labels=all_labels,
objective_field=objective_field)
if args.multi_label:
file_labels = map(slugify,
u.objective_field_names(models_or_ensembles, api))
for index in range(0, len(evaluations)):
evaluation = evaluations[index]
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
if r.shared_changed(args.shared, evaluation):
evaluation_args = {"shared": args.shared}
evaluation = r.update_evaluation(evaluation,
evaluation_args,
args,
api=api,
path=path,
session_file=session_file)
file_name = output
if args.multi_label:
suffix = file_labels[index]
file_name += "_%s" % suffix
evaluation_files.append("%s.json" % file_name)
if args.test_datasets or args.dataset_off:
suffix = evaluation['resource'].replace('evaluation/', '_')
file_name += "_%s" % suffix
evaluation_files.append("%s.json" % file_name)
r.save_evaluation(evaluation, file_name, api)
if args.multi_label or args.test_datasets or args.dataset_off:
mean_evaluation = average_evaluations(evaluation_files)
r.save_evaluation(mean_evaluation, output, api)
return resume
def evaluate(model, dataset, name, description, fields, fields_map, output,
api, args, resume,
session_file=None, path=None, log=None):
"""Evaluates a model or an ensemble with the given dataset
"""
if resume:
message = u.dated("Evaluation not found. Resuming.\n")
resume, evaluation = c.checkpoint(
c.is_evaluation_created, path, debug=args.debug, message=message,
log_file=session_file, console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
if args.ensemble:
model_or_ensemble = args.ensemble
else:
model_or_ensemble = model
evaluation = r.create_evaluation(model_or_ensemble, dataset,
evaluation_args,
args, api, path, session_file,
log)
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
r.save_evaluation(evaluation, output, api)
return resume
def cross_validate(models,
dataset,
number_of_evaluations,
name,
description,
fields,
fields_map,
api,
args,
resume,
session_file=None,
path=None,
log=None):
"""Cross-validates using a MONTE-CARLO variant
"""
existing_evaluations = 0
evaluations = []
if resume:
resume, evaluations = c.checkpoint(c.are_evaluations_created,
path,
number_of_evaluations,
debug=args.debug)
if not resume:
existing_evaluations = len(evaluations)
message = u.dated("Found %s evaluations from %s. Resuming.\n" %
(existing_evaluations, number_of_evaluations))
number_of_evaluations -= existing_evaluations
u.log_message(message,
log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
evaluations.extend(
r.create_evaluations(models, dataset, evaluation_args, args, api,
path, session_file, log,
existing_evaluations))
evaluations_files = []
for evaluation in evaluations:
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
model_id = evaluation['object']['model']
file_name = "%s%s%s__evaluation" % (path, os.sep,
model_id.replace("/", "_"))
evaluations_files.append(file_name + ".json")
r.save_evaluation(evaluation, file_name, api)
cross_validation = average_evaluations(evaluations_files)
file_name = "%s%scross_validation" % (path, os.sep)
r.save_evaluation(cross_validation, file_name, api)
def evaluate(time_series_set,
datasets,
api,
args,
resume,
session_file=None,
path=None,
log=None,
fields=None,
dataset_fields=None,
objective_field=None):
"""Evaluates a list of time-series with the given dataset
"""
output = args.predictions
evaluation_files = []
evaluations, resume = evaluations_process(time_series_set,
datasets,
fields,
dataset_fields,
api,
args,
resume,
session_file=session_file,
path=path,
log=log,
objective_field=objective_field)
for index in range(0, len(evaluations)):
evaluation = evaluations[index]
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
if r.shared_changed(args.shared, evaluation):
evaluation_args = {"shared": args.shared}
evaluation = r.update_evaluation(evaluation,
evaluation_args,
args,
api=api,
path=path,
session_file=session_file)
file_name = output
r.save_evaluation(evaluation, file_name, api)
return resume
def evaluate(models_or_ensembles, datasets, api, args, resume,
session_file=None, path=None, log=None,
fields=None, dataset_fields=None,
labels=None, all_labels=None,
objective_field=None):
"""Evaluates a list of models or ensembles with the given dataset
"""
output = args.predictions
evaluation_files = []
evaluations, resume = evaluations_process(
models_or_ensembles, datasets, fields,
dataset_fields, api, args, resume,
session_file=session_file, path=path, log=log,
labels=labels, all_labels=all_labels, objective_field=objective_field)
if hasattr(args, 'multi_label') and args.multi_label:
file_labels = [slugify(name) for name in
u.objective_field_names(models_or_ensembles, api)]
for index in range(0, len(evaluations)):
evaluation = evaluations[index]
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
if r.shared_changed(args.shared, evaluation):
evaluation_args = {"shared": args.shared}
evaluation = r.update_evaluation(evaluation, evaluation_args,
args, api=api, path=path,
session_file=session_file)
file_name = output
if hasattr(args, 'multi_label') and args.multi_label:
suffix = file_labels[index]
file_name += "_%s" % suffix
evaluation_files.append("%s.json" % file_name)
if args.test_datasets or args.dataset_off:
suffix = evaluation['resource'].replace('evaluation/', '_')
file_name += "_%s" % suffix
evaluation_files.append("%s.json" % file_name)
r.save_evaluation(evaluation, file_name, api)
if (hasattr(args, 'multi_label') and args.multi_label) or \
args.test_datasets or args.dataset_off:
mean_evaluation = average_evaluations(evaluation_files)
r.save_evaluation(mean_evaluation, output, api)
return resume
def cross_validate(models, dataset, number_of_evaluations, name, description,
fields, fields_map, api, args, resume,
session_file=None, path=None, log=None):
"""Cross-validates using a MONTE-CARLO variant
"""
existing_evaluations = 0
evaluations = []
if resume:
resume, evaluations = c.checkpoint(c.are_evaluations_created, path,
number_of_evaluations,
debug=args.debug)
if not resume:
existing_evaluations = len(evaluations)
message = u.dated("Found %s evaluations from %s. Resuming.\n" %
(existing_evaluations,
number_of_evaluations))
number_of_evaluations -= existing_evaluations
u.log_message(message, log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
evaluations.extend(r.create_evaluations(models, dataset,
evaluation_args,
args, api, path,
session_file, log,
existing_evaluations))
evaluations_files = []
for evaluation in evaluations:
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
model_id = evaluation['object']['model']
file_name = "%s%s%s__evaluation" % (path, os.sep,
model_id.replace("/", "_"))
evaluations_files.append(file_name + ".json")
r.save_evaluation(evaluation, file_name, api)
cross_validation = average_evaluations(evaluations_files)
file_name = "%s%scross_validation" % (path, os.sep)
r.save_evaluation(cross_validation, file_name, api)
def cross_validate(models, dataset, fields, api, args, resume,
session_file=None, path=None, log=None):
"""Cross-validates using a MONTE-CARLO variant
"""
evaluations, resume = evaluations_process(
models, [dataset],
fields, fields, api, args, resume,
session_file=session_file, path=path, log=log)
if not resume:
evaluation_files = []
for evaluation in evaluations:
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
model_id = evaluation['object']['model']
file_name = "%s%s%s__evaluation" % (path, os.sep,
model_id.replace("/", "_"))
evaluation_files.append(file_name + ".json")
r.save_evaluation(evaluation, file_name, api)
cross_validation = average_evaluations(evaluation_files)
file_name = "%s%scross_validation" % (path, os.sep)
r.save_evaluation(cross_validation, file_name, api)
resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
debug=args.debug)
if not resume:
message = u.dated("Evaluation not found. Resuming.\n")
u.log_message(message, log_file=session_file,
console=args.verbosity)
if not resume:
evaluation_args = r.set_evaluation_args(name, description, args,
fields, fields_map)
evaluation = r.create_evaluation(model, dataset, evaluation_args,
args, api, path, session_file,
log)
evaluation = r.get_evaluation(evaluation, api, args.verbosity,
session_file)
r.save_evaluation(evaluation, output, api)
# Workaround to restore windows console cp850 encoding to print the tree
if sys.platform == "win32" and sys.stdout.isatty():
import locale
data_locale = locale.getlocale()
if not data_locale[0] is None:
locale.setlocale(locale.LC_ALL, (data_locale[0], "850"))
message = (u"\nGenerated files:\n\n" +
unicode(u.print_tree(path, " "), "utf-8") + u"\n")
else:
message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
u.log_message(message, log_file=session_file, console=args.verbosity)
def main(args=sys.argv[1:]):
