Example #1
0
def evaluate(time_series_set, datasets, api, args, resume,
             session_file=None, path=None, log=None,
             fields=None, dataset_fields=None,
             objective_field=None):
    """Evaluates a list of time-series with the given dataset

    """
    output = args.predictions
    evaluation_files = []
    evaluations, resume = evaluations_process(
        time_series_set, datasets, fields,
        dataset_fields, api, args, resume,
        session_file=session_file, path=path, log=log,
        objective_field=objective_field)
    for index in range(0, len(evaluations)):
        evaluation = evaluations[index]
        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        if r.shared_changed(args.shared, evaluation):
            evaluation_args = {"shared": args.shared}
            evaluation = r.update_evaluation(evaluation, evaluation_args,
                                             args, api=api, path=path,
                                             session_file=session_file)
        file_name = output
        r.save_evaluation(evaluation, file_name, api)
    return resume
Example #2
0
def cross_validate(models,
                   dataset,
                   fields,
                   api,
                   args,
                   resume,
                   session_file=None,
                   path=None,
                   log=None):
    """Cross-validates using a MONTE-CARLO variant

    """
    evaluations, resume = evaluations_process(models, [dataset],
                                              fields,
                                              fields,
                                              api,
                                              args,
                                              resume,
                                              session_file=session_file,
                                              path=path,
                                              log=log)
    if not resume:
        evaluation_files = []
        for evaluation in evaluations:
            evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                          session_file)
            model_id = evaluation['object']['model']
            file_name = "%s%s%s__evaluation" % (path, os.sep,
                                                model_id.replace("/", "_"))
            evaluation_files.append(file_name + ".json")
            r.save_evaluation(evaluation, file_name, api)
        cross_validation = average_evaluations(evaluation_files)
        file_name = "%s%scross_validation" % (path, os.sep)
        r.save_evaluation(cross_validation, file_name, api)
Example #3
0
def evaluate(model, dataset, name, description, fields, fields_map, output,
             api, args, resume,
             session_file=None, path=None, log=None):
    """Evaluates a model or an ensemble with the given dataset

    """
    if resume:
        message = u.dated("Evaluation not found. Resuming.\n")
        resume, evaluation = c.checkpoint(
            c.is_evaluation_created, path, debug=args.debug, message=message,
            log_file=session_file, console=args.verbosity)

    if not resume:
        evaluation_args = r.set_evaluation_args(name, description, args,
                                                fields, fields_map)
        if args.ensemble:
            model_or_ensemble = args.ensemble
        else:
            model_or_ensemble = model
        evaluation = r.create_evaluation(model_or_ensemble, dataset,
                                         evaluation_args,
                                         args, api, path, session_file,
                                         log)

    evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                  session_file)
    r.save_evaluation(evaluation, output, api)
    return resume
Example #4
0
def evaluate(models_or_ensembles, datasets, output, api, args, resume,
             session_file=None, path=None, log=None, name=None,
             description=None, fields=None, dataset_fields=None,
             fields_map=None, labels=None, all_labels=None,
             objective_field=None):
    """Evaluates a list of models or ensembles with the given dataset

    """
    evaluation_files = []
    evaluations, resume = evaluations_process(
        models_or_ensembles, datasets, name, description, fields,
        dataset_fields, fields_map, api, args, resume,
        session_file=session_file, path=path, log=log,
        labels=labels, all_labels=all_labels, objective_field=objective_field)
    if args.multi_label:
        file_labels = map(slugify,
                          u.objective_field_names(models_or_ensembles, api))
    for index in range(0, len(evaluations)):
        evaluation = evaluations[index]
        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        file_name = output
        if args.multi_label:
            suffix = file_labels[index]
            file_name += "_%s" % suffix
            evaluation_files.append("%s.json" % file_name)
        r.save_evaluation(evaluation, file_name, api)
    if args.multi_label:
        mean_evaluation = average_evaluations(evaluation_files)
        r.save_evaluation(mean_evaluation, output, api)
    return resume
Example #5
0
def evaluate(models_or_ensembles,
             datasets,
             api,
             args,
             resume,
             session_file=None,
             path=None,
             log=None,
             fields=None,
             dataset_fields=None,
             labels=None,
             all_labels=None,
             objective_field=None):
    """Evaluates a list of models or ensembles with the given dataset

    """
    output = args.predictions
    evaluation_files = []
    evaluations, resume = evaluations_process(models_or_ensembles,
                                              datasets,
                                              fields,
                                              dataset_fields,
                                              api,
                                              args,
                                              resume,
                                              session_file=session_file,
                                              path=path,
                                              log=log,
                                              labels=labels,
                                              all_labels=all_labels,
                                              objective_field=objective_field)
    if args.multi_label:
        file_labels = map(slugify,
                          u.objective_field_names(models_or_ensembles, api))
    for index in range(0, len(evaluations)):
        evaluation = evaluations[index]
        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        if r.shared_changed(args.shared, evaluation):
            evaluation_args = {"shared": args.shared}
            evaluation = r.update_evaluation(evaluation,
                                             evaluation_args,
                                             args,
                                             api=api,
                                             path=path,
                                             session_file=session_file)
        file_name = output
        if args.multi_label:
            suffix = file_labels[index]
            file_name += "_%s" % suffix
            evaluation_files.append("%s.json" % file_name)
        if args.test_datasets or args.dataset_off:
            suffix = evaluation['resource'].replace('evaluation/', '_')
            file_name += "_%s" % suffix
            evaluation_files.append("%s.json" % file_name)
        r.save_evaluation(evaluation, file_name, api)
    if args.multi_label or args.test_datasets or args.dataset_off:
        mean_evaluation = average_evaluations(evaluation_files)
        r.save_evaluation(mean_evaluation, output, api)
    return resume
Example #6
0
def evaluate(model, dataset, name, description, fields, fields_map, output,
             api, args, resume,
             session_file=None, path=None, log=None):
    """Evaluates a model or an ensemble with the given dataset

    """
    if resume:
        message = u.dated("Evaluation not found. Resuming.\n")
        resume, evaluation = c.checkpoint(
            c.is_evaluation_created, path, debug=args.debug, message=message,
            log_file=session_file, console=args.verbosity)

    if not resume:
        evaluation_args = r.set_evaluation_args(name, description, args,
                                                fields, fields_map)
        if args.ensemble:
            model_or_ensemble = args.ensemble
        else:
            model_or_ensemble = model
        evaluation = r.create_evaluation(model_or_ensemble, dataset,
                                         evaluation_args,
                                         args, api, path, session_file,
                                         log)

    evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                  session_file)
    r.save_evaluation(evaluation, output, api)
    return resume
Example #7
0
def cross_validate(models,
                   dataset,
                   number_of_evaluations,
                   name,
                   description,
                   fields,
                   fields_map,
                   api,
                   args,
                   resume,
                   session_file=None,
                   path=None,
                   log=None):
    """Cross-validates using a MONTE-CARLO variant

    """
    existing_evaluations = 0
    evaluations = []
    if resume:
        resume, evaluations = c.checkpoint(c.are_evaluations_created,
                                           path,
                                           number_of_evaluations,
                                           debug=args.debug)
        if not resume:
            existing_evaluations = len(evaluations)
            message = u.dated("Found %s evaluations from %s. Resuming.\n" %
                              (existing_evaluations, number_of_evaluations))
            number_of_evaluations -= existing_evaluations
            u.log_message(message,
                          log_file=session_file,
                          console=args.verbosity)
    if not resume:
        evaluation_args = r.set_evaluation_args(name, description, args,
                                                fields, fields_map)

        evaluations.extend(
            r.create_evaluations(models, dataset, evaluation_args, args, api,
                                 path, session_file, log,
                                 existing_evaluations))
        evaluations_files = []
        for evaluation in evaluations:
            evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                          session_file)
            model_id = evaluation['object']['model']
            file_name = "%s%s%s__evaluation" % (path, os.sep,
                                                model_id.replace("/", "_"))
            evaluations_files.append(file_name + ".json")
            r.save_evaluation(evaluation, file_name, api)
        cross_validation = average_evaluations(evaluations_files)
        file_name = "%s%scross_validation" % (path, os.sep)
        r.save_evaluation(cross_validation, file_name, api)
Example #8
0
def evaluate(time_series_set,
             datasets,
             api,
             args,
             resume,
             session_file=None,
             path=None,
             log=None,
             fields=None,
             dataset_fields=None,
             objective_field=None):
    """Evaluates a list of time-series with the given dataset

    """
    output = args.predictions
    evaluation_files = []
    evaluations, resume = evaluations_process(time_series_set,
                                              datasets,
                                              fields,
                                              dataset_fields,
                                              api,
                                              args,
                                              resume,
                                              session_file=session_file,
                                              path=path,
                                              log=log,
                                              objective_field=objective_field)
    for index in range(0, len(evaluations)):
        evaluation = evaluations[index]
        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        if r.shared_changed(args.shared, evaluation):
            evaluation_args = {"shared": args.shared}
            evaluation = r.update_evaluation(evaluation,
                                             evaluation_args,
                                             args,
                                             api=api,
                                             path=path,
                                             session_file=session_file)
        file_name = output
        r.save_evaluation(evaluation, file_name, api)
    return resume
Example #9
0
def evaluate(models_or_ensembles, datasets, api, args, resume,
             session_file=None, path=None, log=None,
             fields=None, dataset_fields=None,
             labels=None, all_labels=None,
             objective_field=None):
    """Evaluates a list of models or ensembles with the given dataset

    """
    output = args.predictions
    evaluation_files = []
    evaluations, resume = evaluations_process(
        models_or_ensembles, datasets, fields,
        dataset_fields, api, args, resume,
        session_file=session_file, path=path, log=log,
        labels=labels, all_labels=all_labels, objective_field=objective_field)
    if hasattr(args, 'multi_label') and args.multi_label:
        file_labels = [slugify(name) for name in
                       u.objective_field_names(models_or_ensembles, api)]
    for index in range(0, len(evaluations)):
        evaluation = evaluations[index]
        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        if r.shared_changed(args.shared, evaluation):
            evaluation_args = {"shared": args.shared}
            evaluation = r.update_evaluation(evaluation, evaluation_args,
                                             args, api=api, path=path,
                                             session_file=session_file)
        file_name = output
        if hasattr(args, 'multi_label') and args.multi_label:
            suffix = file_labels[index]
            file_name += "_%s" % suffix
            evaluation_files.append("%s.json" % file_name)
        if args.test_datasets or args.dataset_off:
            suffix = evaluation['resource'].replace('evaluation/', '_')
            file_name += "_%s" % suffix
            evaluation_files.append("%s.json" % file_name)
        r.save_evaluation(evaluation, file_name, api)
    if (hasattr(args, 'multi_label') and args.multi_label) or \
            args.test_datasets or args.dataset_off:
        mean_evaluation = average_evaluations(evaluation_files)
        r.save_evaluation(mean_evaluation, output, api)
    return resume
Example #10
0
def cross_validate(models, dataset, number_of_evaluations, name, description,
                   fields, fields_map, api, args, resume,
                   session_file=None, path=None, log=None):
    """Cross-validates using a MONTE-CARLO variant

    """
    existing_evaluations = 0
    evaluations = []
    if resume:
        resume, evaluations = c.checkpoint(c.are_evaluations_created, path,
                                           number_of_evaluations,
                                           debug=args.debug)
        if not resume:
            existing_evaluations = len(evaluations)
            message = u.dated("Found %s evaluations from %s. Resuming.\n" %
                              (existing_evaluations,
                               number_of_evaluations))
            number_of_evaluations -= existing_evaluations
            u.log_message(message, log_file=session_file,
                          console=args.verbosity)
    if not resume:
        evaluation_args = r.set_evaluation_args(name, description, args,
                                                fields, fields_map)

        evaluations.extend(r.create_evaluations(models, dataset,
                                                evaluation_args,
                                                args, api, path,
                                                session_file, log,
                                                existing_evaluations))
        evaluations_files = []
        for evaluation in evaluations:
            evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                          session_file)
            model_id = evaluation['object']['model']
            file_name = "%s%s%s__evaluation" % (path, os.sep,
                                                model_id.replace("/", "_"))
            evaluations_files.append(file_name + ".json")
            r.save_evaluation(evaluation, file_name, api)
        cross_validation = average_evaluations(evaluations_files)
        file_name = "%s%scross_validation" % (path, os.sep)
        r.save_evaluation(cross_validation, file_name, api)
Example #11
0
def cross_validate(models, dataset, fields, api, args, resume,
                   session_file=None, path=None, log=None):
    """Cross-validates using a MONTE-CARLO variant

    """
    evaluations, resume = evaluations_process(
        models, [dataset],
        fields, fields, api, args, resume,
        session_file=session_file, path=path, log=log)
    if not resume:
        evaluation_files = []
        for evaluation in evaluations:
            evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                          session_file)
            model_id = evaluation['object']['model']
            file_name = "%s%s%s__evaluation" % (path, os.sep,
                                                model_id.replace("/", "_"))
            evaluation_files.append(file_name + ".json")
            r.save_evaluation(evaluation, file_name, api)
        cross_validation = average_evaluations(evaluation_files)
        file_name = "%s%scross_validation" % (path, os.sep)
        r.save_evaluation(cross_validation, file_name, api)
Example #12
0
            resume, evaluation = u.checkpoint(u.is_evaluation_created, path,
                                              debug=args.debug)
            if not resume:
                message = u.dated("Evaluation not found. Resuming.\n")
                u.log_message(message, log_file=session_file,
                              console=args.verbosity)
        if not resume:
            evaluation_args = r.set_evaluation_args(name, description, args,
                                                    fields, fields_map)
            evaluation = r.create_evaluation(model, dataset, evaluation_args,
                                             args, api, path, session_file,
                                             log)

        evaluation = r.get_evaluation(evaluation, api, args.verbosity,
                                      session_file)
        r.save_evaluation(evaluation, output, api)

    # Workaround to restore windows console cp850 encoding to print the tree
    if sys.platform == "win32" and sys.stdout.isatty():
        import locale
        data_locale = locale.getlocale()
        if not data_locale[0] is None:
            locale.setlocale(locale.LC_ALL, (data_locale[0], "850"))
        message = (u"\nGenerated files:\n\n" +
                   unicode(u.print_tree(path, " "), "utf-8") + u"\n")
    else:
        message = "\nGenerated files:\n\n" + u.print_tree(path, " ") + "\n"
    u.log_message(message, log_file=session_file, console=args.verbosity)


def main(args=sys.argv[1:]):