def i_create_cluster_resources_from_local_cluster(step, directory=None, test=None, output=None):
    ok_(test is not None and output is not None and directory is not None)
    test = res_filename(test)
    with open(os.path.join(directory, "clusters")) as handler:
        cluster_id = handler.readline().strip()
    command = ("bigmler cluster --cluster-file " +
               storage_file_name(directory, cluster_id) +
               " --test " + test +
               " --store --output " + output)
    shell_execute(command, output, test=test)
def i_create_anomaly_resources_from_local_anomaly_detector(step, directory=None, test=None, output=None):
    if test is None or output is None or directory is None:
        assert False
    with open(os.path.join(directory, "anomalies")) as handler:
        anomaly_id = handler.readline().strip()
    command = ("bigmler anomaly --anomaly-file " +
               storage_file_name(directory, anomaly_id) +
               " --test " + test +
               " --store --output " + output)
    shell_execute(command, output, test=test)
Exemple #3
0
def i_create_anomaly_resources_from_local_anomaly_detector(
        step, directory=None, test=None, output=None):
    if test is None or output is None or directory is None:
        assert False
    with open(os.path.join(directory, "anomalies")) as handler:
        anomaly_id = handler.readline().strip()
    command = ("bigmler anomaly --anomaly-file " +
               storage_file_name(directory, anomaly_id) + " --test " + test +
               " --store --output " + output)
    shell_execute(command, output, test=test)
Exemple #4
0
def i_create_cluster_resources_from_local_cluster(step,
                                                  directory=None,
                                                  test=None,
                                                  output=None):
    ok_(test is not None and output is not None and directory is not None)
    test = res_filename(test)
    with open(os.path.join(directory, "clusters")) as handler:
        cluster_id = handler.readline().strip()
    command = ("bigmler cluster --cluster-file " +
               storage_file_name(directory, cluster_id) + " --test " + test +
               " --store --output " + output)
    shell_execute(command, output, test=test)
Exemple #5
0
def best_first_search(datasets_file,
                      api,
                      args,
                      common_options,
                      staleness=None,
                      penalty=None,
                      objective_name=None,
                      resume=False):
    """Selecting the fields to be used in the model construction

    """
    counter = 0
    loop_counter = 0
    features_file = os.path.normpath(
        os.path.join(args.output_dir, FEATURES_LOG))
    with open(features_file, u.open_mode("w")) as features_handler:
        features_writer = csv.writer(features_handler, lineterminator="\n")
        features_writer.writerow(
            ["step", "state", "score", "metric_value", "best_score"])
        features_handler.flush()
        if staleness is None:
            staleness = DEFAULT_STALENESS
        if penalty is None:
            penalty = DEFAULT_PENALTY
        # retrieving the first dataset in the file
        try:
            with open(datasets_file, u.open_mode("r")) as datasets_handler:
                dataset_id = datasets_handler.readline().strip()
        except IOError, exc:
            sys.exit("Could not read the generated datasets file: %s" %
                     str(exc))
        try:
            stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
            with open(stored_dataset, u.open_mode("r")) as dataset_handler:
                dataset = json.loads(dataset_handler.read())
        except IOError:
            dataset = api.check_resource(dataset_id,
                                         query_string=ALL_FIELDS_QS)
        # initial feature set
        fields = Fields(dataset)
        excluded_features = ([] if args.exclude_features is None else
                             args.exclude_features.split(args.args_separator))
        try:
            excluded_ids = [
                fields.field_id(feature) for feature in excluded_features
            ]
            objective_id = fields.field_id(objective_name)
        except ValueError, exc:
            sys.exit(exc)
Exemple #6
0
def best_first_search(datasets_file, api, args, common_options,
                      staleness=None, penalty=None, objective_name=None,
                      resume=False):
    """Selecting the fields to be used in the model construction

    """
    counter = 0
    loop_counter = 0
    features_file = os.path.normpath(os.path.join(args.output_dir,
                                                  FEATURES_LOG))
    with open(features_file, u.open_mode("w")) as features_handler:
        features_writer = csv.writer(features_handler, lineterminator="\n")
        features_writer.writerow([
            "step", "state", "score", "metric_value", "best_score"])
        features_handler.flush()
        if staleness is None:
            staleness = DEFAULT_STALENESS
        if penalty is None:
            penalty = DEFAULT_PENALTY
        # retrieving the first dataset in the file
        try:
            with open(datasets_file, u.open_mode("r")) as datasets_handler:
                dataset_id = datasets_handler.readline().strip()
        except IOError, exc:
            sys.exit("Could not read the generated datasets file: %s" %
                     str(exc))
        try:
            stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
            with open(stored_dataset, u.open_mode("r")) as dataset_handler:
                dataset = json.loads(dataset_handler.read())
        except IOError:
            dataset = api.check_resource(dataset_id,
                                         query_string=ALL_FIELDS_QS)
        # initial feature set
        fields = Fields(dataset)
        excluded_features = ([] if args.exclude_features is None else
                             args.exclude_features.split(
                                 args.args_separator))
        try:
            excluded_ids = [fields.field_id(feature) for
                            feature in excluded_features]
            objective_id = fields.field_id(objective_name)
        except ValueError, exc:
            sys.exit(exc)
Exemple #7
0
                                               FEATURES_LOG))
 features_writer = UnicodeWriter(features_file).open_writer()
 features_header = FEATURES_HEADER
 if staleness is None:
     staleness = DEFAULT_STALENESS
 if penalty is None:
     penalty = DEFAULT_PENALTY
 # retrieving the first dataset in the file
 try:
     with open(datasets_file, u.open_mode("r")) as datasets_handler:
         dataset_id = datasets_handler.readline().strip()
 except IOError, exc:
     sys.exit("Could not read the generated datasets file: %s" %
              str(exc))
 try:
     stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
     with open(stored_dataset, u.open_mode("r")) as dataset_handler:
         dataset = json.loads(dataset_handler.read())
 except IOError:
     dataset = api.check_resource(dataset_id,
                                  query_string=ALL_FIELDS_QS)
 # initial feature set
 fields = Fields(dataset)
 excluded_features = ([] if args.exclude_features is None else
                      args.exclude_features.split(
                          args.args_separator))
 try:
     excluded_ids = [fields.field_id(feature) for
                     feature in excluded_features]
     objective_id = fields.field_id(objective_name)
 except ValueError, exc:
Exemple #8
0
 features_file = os.path.normpath(
     os.path.join(args.output_dir, FEATURES_LOG))
 features_writer = UnicodeWriter(features_file).open_writer()
 features_header = FEATURES_HEADER
 if staleness is None:
     staleness = DEFAULT_STALENESS
 if penalty is None:
     penalty = DEFAULT_PENALTY
 # retrieving the first dataset in the file
 try:
     with open(datasets_file, u.open_mode("r")) as datasets_handler:
         dataset_id = datasets_handler.readline().strip()
 except IOError, exc:
     sys.exit("Could not read the generated datasets file: %s" % str(exc))
 try:
     stored_dataset = u.storage_file_name(args.output_dir, dataset_id)
     with open(stored_dataset, u.open_mode("r")) as dataset_handler:
         dataset = json.loads(dataset_handler.read())
 except IOError:
     dataset = api.check_resource(dataset_id, query_string=ALL_FIELDS_QS)
 # initial feature set
 fields = Fields(dataset)
 excluded_features = ([] if args.exclude_features is None else
                      args.exclude_features.split(args.args_separator))
 try:
     excluded_ids = [
         fields.field_id(feature) for feature in excluded_features
     ]
     objective_id = fields.field_id(objective_name)
 except ValueError, exc:
     sys.exit(exc)