def transform(input_path, features_path, labels_path, metadata_path, scaler_path): input_file = open(input_path, "r") reader = csv.DictReader(input_file) variables = set(reader.fieldnames) variables.remove("ID") variables.remove("default payment next month") metadata = create_metadata(variables, TYPES, VALUES, NUM_SAMPLES, CLASSES) # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.int32) # transform sample_index = 0 for row in reader: for variable in metadata["variables"]: value = row[variable] if TYPES[variable] == "numerical": value = float(value) features[sample_index, metadata["value_to_index"][variable]] = value elif TYPES[variable] == "categorical": value = int(float(value)) assert value in ORIGINAL_ENCODING_TO_VALUES[variable], \ "'{}' is not a valid value for '{}'".format(value, variable) value = ORIGINAL_ENCODING_TO_VALUES[variable][value] features[sample_index, metadata["value_to_index"][variable][value]] = 1.0 # the class needs to be transformed labels[sample_index] = int(row["default payment next month"].replace( ".0", "")) # next row sample_index += 1 # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # validate the known distributions validate_num_samples(metadata, sample_index) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(input_path, features_path, labels_path, metadata_path, scaler_path): metadata = create_metadata( VARIABLES, create_one_type_dictionary("numerical", VARIABLES), {}, sum(NUM_SAMPLES), CLASSES) input_file = open(input_path, "r") # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.int32) # transform sample_index = 0 line = input_file.readline() while line != "": line = line.rstrip("\n") values = line.split(",") # the amount of values (minus the label) # should be the same as the amount of variables assert len(values) - 1 == len(VARIABLES), str( (len(values) - 1, len(VARIABLES))) # assign each column with the right mapping for variable, value in zip(VARIABLES, values[:-1]): value = float(value) features[sample_index, metadata["value_to_index"][variable]] = value # the last value is the label labels[sample_index] = int(values[-1]) # next row sample_index += 1 # next line line = input_file.readline() # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # validate the known distributions validate_num_samples(metadata, sample_index) validate_class_distribution(metadata, NUM_SAMPLES) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(input_path, features_path, labels_path, metadata_path, scaler_path): metadata = create_metadata( VARIABLES, create_one_type_dictionary("numerical", VARIABLES), {}, sum(NUM_SAMPLES), CLASSES) input_file = open(input_path, "r") # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.int32) # transform sample_index = 0 line = input_file.readline() while line != "": line = line.rstrip("\n") values = line.split(",") assert len(values) - 1 == len(VARIABLES), str( (len(values) - 1, len(VARIABLES))) for feature_index, value in enumerate(values[1:]): value = float(value) features[sample_index, feature_index] = value labels[sample_index] = CLASS_TO_INDEX[values[0]] # next line line = input_file.readline() # next row sample_index += 1 # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # validate the known distributions validate_num_samples(metadata, sample_index) validate_class_distribution(metadata, NUM_SAMPLES) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(train_path, test_path, features_path, labels_path, metadata_path, ignore_missing, scaler_path): num_samples_train = NUM_SAMPLES[ignore_missing]["train"] num_samples_test = NUM_SAMPLES[ignore_missing]["test"] metadata = create_metadata(VARIABLES, TYPES, VALUES, num_samples_train + num_samples_test, CLASSES) # transform train train_file = open(train_path, "r") features_train, labels_train = adult_transform_file( train_file, num_samples_train, metadata["num_features"], metadata["value_to_index"], ignore_missing) train_file.close() # transform test test_file = open(test_path, "r") test_file.readline() # has an extra first line features_test, labels_test = adult_transform_file( test_file, num_samples_test, metadata["num_features"], metadata["value_to_index"], ignore_missing) test_file.close() # concatenate train and test features = np.concatenate((features_train, features_test)) labels = np.concatenate((labels_train, labels_test)) # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # save np.save(features_path, features) np.save(labels_path, labels) with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(input_path, features_path, labels_path, metadata_path, ignore_missing, scaler_path): metadata = create_metadata( VARIABLES, create_one_type_dictionary("numerical", VARIABLES), {}, sum(NUM_SAMPLES[ignore_missing]), CLASSES) input_file = open(input_path, "r") # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.int32) # transform sample_index = 0 line = input_file.readline() while line != "": line = line.rstrip("\n") values = line.split(",") missing_values = False for value in values: if value == "?" or value == "": missing_values = True break # if there are missing values the ignore missing flag is set then ignore the row if not missing_values or not ignore_missing: # the amount of values (minus the ID and the label) # should be the same as the amount of variables assert len(values) - 2 == len(VARIABLES), str( (len(values), len(VARIABLES))) # assign each column with the right mapping # (skip the ID and the label) for variable, value in zip(VARIABLES, values[1:-1]): if value == "?" or value == "": value = np.nan else: value = float(value) features[sample_index, metadata["value_to_index"][variable]] = value # the second value is the label labels[sample_index] = CLASS_TO_INDEX[values[-1]] # next row sample_index += 1 # next line line = input_file.readline() # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # validate the known distributions validate_num_samples(metadata, sample_index) validate_class_distribution(metadata, NUM_SAMPLES[ignore_missing]) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(input_path, features_path, labels_path, metadata_path, scaler_path): metadata = create_metadata(VARIABLES, TYPES, VALUES, sum(NUM_SAMPLES), CLASSES) # The raw data is already nicely one-hot-encoded, but we need to follow the standard of our metadata. # The most important thing is that we put the categorical features at the beginning. # The extra annoying thing is that we also sort the categorical values alphabetically. # Because of this, we need to do map of feature indices unnecessarily. old_to_new_feature_indices = {} old_feature_index = 0 for variable in VARIABLES: if TYPES[variable] == "numerical": old_to_new_feature_indices[old_feature_index] = metadata["value_to_index"][variable] old_feature_index += 1 elif TYPES[variable] == "categorical": for value in VALUES[variable]: old_to_new_feature_indices[old_feature_index] = metadata["value_to_index"][variable][value] old_feature_index += 1 else: raise Exception("Invalid type.") input_file = open(input_path, "r") # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.int32) # transform sample_index = 0 line = input_file.readline() while line != "": line = line.rstrip("\n") values = line.split(",") # transform original class numbers to 0 indexed arrays class_number = int(values[-1]) - 1 if sample_index < metadata["num_samples"]: # the categorical variables are already one hot encoded for old_feature_index, value in enumerate(values[:-1]): new_feature_index = old_to_new_feature_indices[old_feature_index] value = float(value) features[sample_index, new_feature_index] = value # the class needs to be transformed labels[sample_index] = class_number # next line line = input_file.readline() # next row sample_index += 1 # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) update_class_distribution(metadata, labels) # validate the known distributions validate_num_samples(metadata, sample_index) validate_class_distribution(metadata, NUM_SAMPLES) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)
def transform(input_path, features_path, labels_path, metadata_path, scaler_path): metadata = create_metadata(VARIABLES, TYPES, VALUES, NUM_SAMPLES) input_file = open(input_path, "r") reader = csv.DictReader(input_file) reader.fieldnames = [variable.strip() for variable in reader.fieldnames] # initialize outputs features = np.zeros((metadata["num_samples"], metadata["num_features"]), dtype=np.float32) labels = np.zeros(metadata["num_samples"], dtype=np.float32) # transform sample_index = 0 for row in reader: for variable in metadata["variables"]: # numerical variable if TYPES[variable] == "numerical": value = float(row[variable]) features[sample_index, metadata["value_to_index"][variable]] = value # categorical variable elif TYPES[variable] == "categorical": value = None # check all possible values for possible_value in VALUES[variable]: # skip the none possible value if possible_value == "none": continue # if the possible value binary flag is set real_variable = "{}{}{}".format( variable, ONE_HOT_ENCODING_SEPARATOR, possible_value) if read_binary(row[real_variable]) == 1: # if the categorical variable had no value set if value is None: value = possible_value # the categorical variable already had a value set else: raise Exception( "'{}' was already defined".format(variable)) # if no binary flag was set for the categorical variable if value is None: # if it is possible to have no value if "none" in VALUES[variable]: value = "none" # if it is not possible to have no value but there is no value else: raise Exception( "'{}' has no valid value".format(variable)) # set the categorical variable flag in the mapped feature features[sample_index, metadata["value_to_index"][variable][value]] = 1.0 # binary variable elif TYPES[variable] == "binary": value = read_binary(row[variable]) assert value in [ 0, 1 ], "'{}' is not a valid value for '{}'".format( value, variable) features[sample_index, metadata["value_to_index"][variable]] = value # unknown variable type else: raise Exception("Unknown variable type.") # label labels[sample_index] = row["shares"] # next row sample_index += 1 # scale if scaler_path is not None: features = scale_and_save_scaler(features, scaler_path) # add distributions to the metadata update_feature_distributions(metadata, features) # validate the known distributions validate_num_samples(metadata, sample_index) np.save(features_path, features) np.save(labels_path, labels) input_file.close() with open(metadata_path, "w") as metadata_file: json.dump(metadata, metadata_file)