def _encode_numerical_feature( feature: KerasTensor, name: str, dataset: Optional[BatchDataset], ) -> KerasTensor: """Normalize numerical features. Args: - feature: The input layer of the feature. - name: The feature's name (its column name in the original dataframe). - dataset: The training data, if not specified, return a no-op layer. Returns: The normalized tensor of the input feature. """ # Return generic layer for the tuner initialization if not dataset: return KerasTensor(type_spec=TensorSpec( shape=(None, 1), dtype=tf.float32, name=None)) # Create a Normalization layer for our feature normalizer = Normalization() # Prepare a Dataset that only yields our feature feature_ds = dataset.map(lambda x, y: x[name]) feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1)) # Learn the statistics of the data normalizer.adapt(feature_ds) # Normalize the input feature encoded_feature = normalizer(feature) return encoded_feature
def cifar_standardization(x, mode='FEATURE_NORMALIZE', data_samples=None): mode = mode.upper() assert mode in ['FEATURE_NORMALIZE', 'PIXEL_MEAN_SUBTRACT'] if mode == 'PIXEL_MEAN_SUBTRACT' and not data_samples: raise ValueError('`data_samples` argument should not be `None`, ' 'when `mode="PIXEL_MEAN_SUBTRACT"`.') if mode == 'FEATURE_NORMALIZE': cifar_mean = tf.cast(CIFAR_MEAN, tf.float32).numpy() cifar_std = tf.cast(CIFAR_STD, tf.float32).numpy() x = Rescaling(scale=1. / cifar_std, offset=-(cifar_mean / cifar_std), name='mean_normalization')(x) elif mode == 'PIXEL_MEAN_SUBTRACT': mean_subtraction_layer = Normalization(axis=[1, 2, 3], name='pixel_mean_subtraction') mean_subtraction_layer.adapt(data_samples) # set values of variance = 1. and keep mean values as is mean_pixels = mean_subtraction_layer.get_weights()[0] mean_subtraction_layer.set_weights( [mean_pixels, tf.ones_like(mean_pixels)]) x = mean_subtraction_layer(x) x = Rescaling(scale=1 / 255., name='rescaling')(x) return x
def min_normalizer(): """ Normalizer with minimalistic data.""" adapt_data = np.array([ [1., 2.], [2., 3.], ], dtype=np.float32) normalizer = Normalization() normalizer.adapt(adapt_data) return normalizer
def encode_numerical_feature(feature, name, dataset): normalizer = Normalization() feature_ds = dataset.map(lambda x, y: x[name]) feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1)) normalizer.adapt(feature_ds) encoded_feature = normalizer(feature) return encoded_feature
def encode_numerical_feature(feature, name, dataset): # Create a Keras Normalization Layer for the input feature passed as argument normalizer = Normalization() # Prepare a Dataset containing only the feature feature_dset = dataset.map(lambda x, y: x[name]) feature_dset = feature_dset.map(lambda x: tf.expand_dims(x, -1)) # Learn the statistics of the data and normalise the input feature normalizer.adapt(feature_dset) encoded_feature = normalizer(feature) return encoded_feature
def load_data(): data = pd.read_csv( "https://storage.googleapis.com/tf-datasets/titanic/train.csv") from sklearn.model_selection import train_test_split labels = data.pop('survived') label_names = ["Not survived", "Survived"] features = {} # Converting CSV file into Tensorflow object for name, column in data.items(): dtype = column.dtype if dtype == object: dtype = string else: dtype = float32 features[name] = Input(shape=(1, ), name=name, dtype=dtype) # Extracting and normalizing numeric features numeric_features = { name: feature for name, feature in features.items() if feature.dtype == float32 } x = Concatenate()(list(numeric_features.values())) norm = Normalization() norm.adapt(np.array(data[numeric_features.keys()])) numeric_features = norm(x) processed_features = [numeric_features] # Extracting and normalizing non-numeric features for name, feature in features.items(): if feature.dtype == float32: continue word = StringLookup(vocabulary=np.unique(data[name])) one_hot = CategoryEncoding(max_tokens=word.vocab_size()) x = word(feature) x = one_hot(x) processed_features.append(x) processed_features = Concatenate()(processed_features) processed_features = Model(features, processed_features) utils.plot_model(model=processed_features, rankdir='LR', dpi=72, show_shapes=True) feature_dict = {name: np.array(value) for name, value in data.items()} train_features, test_features, train_labels, test_labels = train_test_split( processed_features(feature_dict).numpy(), labels, test_size=0.2) return train_features, train_labels, test_features, test_labels
def encodeMyFeature(indidualFeature, name, dataset): # Normalization the data normalizer = Normalization() # Pull out a data set for each feature. feature_ds = dataset.map(lambda x, y: x[name]) feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1)) # built in code to find the statistics of the data normalizer.adapt(feature_ds) encodedFeature = normalizer(indidualFeature) return encodedFeature
def encode_numerical_feature(feature, name, dataset): # Create a Normalization layer for our feature normalizer = Normalization() # Prepare a Dataset that only yields our feature feature_ds = dataset.map(lambda x, y: x[name]) feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1)) # Learn the statistics of the data normalizer.adapt(feature_ds) # Normalize the input feature encoded_feature = normalizer(feature) return encoded_feature
class Data: def __init__(self, filename, m_input='u', m_output='y'): # Load data data = loadmat(filename) # Normalize input data self.input = data[m_input] self.layernorminput = Normalization() self.layernorminput.adapt(self.input) self.input_norm = self.layernorminput(self.input) # Normalize output data self.output = data[m_output] self.layernormoutput = Normalization() self.layernormoutput.adapt(self.output) self.output_norm = self.layernormoutput(self.output)
def train(data): types = set() for file in data["files"]: types.add(file["type"]) # Try to get >100 of each type training_input, training_output = convert_data_to_np(data, list(types)) # n = tf.keras.utils.normalize(training_input, axis=-1, order=2) normalizer = Normalization(axis=-1) normalizer.adapt(training_input) normalized_data = normalizer(training_input) print("var: %.4f" % np.var(normalized_data)) print("mean: %.4f" % np.mean(normalized_data)) dense = keras.layers.Dense(units=16)
# token. integer_data = vectorizer(training_data) print(integer_data) """ **Example: normalizing features** """ from tensorflow.keras.layers.experimental.preprocessing import Normalization # Example image data, with values in the [0, 255] range training_data = np.random.randint(0, 256, size=(64, 200, 200, 3)).astype("float32") normalizer = Normalization(axis=-1) normalizer.adapt(training_data) normalized_data = normalizer(training_data) print("var: %.4f" % np.var(normalized_data)) print("mean: %.4f" % np.mean(normalized_data)) """ **Example: rescaling & center-cropping images** Both the `Rescaling` layer and the `CenterCrop` layer are stateless, so it isn't necessary to call `adapt()` in this case. """ from tensorflow.keras.layers.experimental.preprocessing import CenterCrop from tensorflow.keras.layers.experimental.preprocessing import Rescaling # Example image data, with values in the [0, 255] range
""" # normalize in range [0, 1] scaling_layer = Rescaling(1.0 / 255) # normalize in range [-1, 1] input_ = tf.keras.Input(shape=(32, 32, 3)) norm_neg_one_to_one = Normalization() x = norm_neg_one_to_one(input_) import numpy as np mean = [127.5]*3 var = mean ** 2 norm_neg_one_to_one.set_weights([mean, var]) norm_neg_one_to_one.get_weights() # normalize with mean 0 and std 1 norm_mean_std = Normalization() norm_mean_std.adapt(x_train[0]) model_ = Sequential([ tf.keras.Input(shape=(32, 32, 3)), norm_mean_std, model ]) model_.compile( optimizer="Adam", loss="sparse_categorical_crossentropy", metrics=['accuracy'], ) model_.fit(x_train, y_train, epochs=1, batch_size=16) """ When you don't have a large image dataset, it's a good practice
class CALIHOUSING: def __init__(self, test_size: float = 0.2, validation_size: float = 0.33) -> None: # User-definen constants self.num_targets = 1 self.batch_size = 128 # Load the data set dataset = fetch_california_housing() self.x, self.y = dataset.data, dataset.target self.feature_names = dataset.feature_names self.description = dataset.DESCR # Split the dataset x_train, x_test, y_train, y_test = train_test_split( self.x, self.y, test_size=test_size) x_train, x_val, y_train, y_val = train_test_split( x_train, y_train, test_size=validation_size) # Preprocess x data self.x_train = x_train.astype(np.float32) self.x_test = x_test.astype(np.float32) self.x_val = x_val.astype(np.float32) # Preprocess y data self.y_train = np.reshape(y_train, (-1, self.num_targets)).astype(np.float32) self.y_test = np.reshape(y_test, (-1, self.num_targets)).astype(np.float32) self.y_val = np.reshape(y_val, (-1, self.num_targets)).astype(np.float32) # Dataset attributes self.train_size = self.x_train.shape[0] self.test_size = self.x_test.shape[0] self.num_features = self.x_train.shape[1] self.num_targets = self.y_train.shape[1] # Normalization variables self.normalization_layer = Normalization() self.normalization_layer.adapt(self.x_train) # tf.data Datasets self.train_dataset = tf.data.Dataset.from_tensor_slices( (self.x_train, self.y_train)) self.test_dataset = tf.data.Dataset.from_tensor_slices( (self.x_test, self.y_test)) self.val_dataset = tf.data.Dataset.from_tensor_slices( (self.x_val, self.y_val)) self.train_dataset = self._prepare_dataset(self.train_dataset, shuffle=True) self.test_dataset = self._prepare_dataset(self.test_dataset) self.val_dataset = self._prepare_dataset(self.val_dataset) def get_train_set(self) -> tf.data.Dataset: return self.train_dataset def get_test_set(self) -> tf.data.Dataset: return self.test_dataset def get_val_set(self) -> tf.data.Dataset: return self.val_dataset def _prepare_dataset(self, dataset: tf.data.Dataset, shuffle: bool = False) -> tf.data.Dataset: dataset = dataset.map( map_func=lambda x, y: (tf.reshape(self.normalization_layer( tf.reshape(x, shape=(1, self.num_features)), training=False), shape=(self.num_features, )), y), num_parallel_calls=tf.data.experimental.AUTOTUNE) if shuffle: dataset = dataset.shuffle(buffer_size=1_000) dataset = dataset.batch(batch_size=self.batch_size) return dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
print(dept_targets) dataset = keras.preprocessing.image_dataset_from_directory('mian_directory', batch_size=64, image_size=(2000, 2000)) for data, labels in dataset: print(data.shape) print(data.dtype) print(labels.shape) print(labels.dtype) #规范化 trianing_data = np.random.randint(0, 256, size=(64, 200, 200, 3)).astype("float32") normalizetion = Normalization(axis=-1) #沿着最后一个下标变换的方向 normalizetion.adapt(trianing_data) normalizetion_data = normalizetion(trianing_data) print("var:%.4f" % np.var(normalizetion_data)) #np.var求标准差 print("mean:%.4f" % np.mean(normalizetion_data)) #重新缩放和中心裁剪图像 cropper = CenterCrop(height=150, width=150) scaler = Rescaling(scale=1.0 / 255) output_data = scaler(cropper( trianing_data)) #无论是Rescaling层与CenterCrop层是无状态的,所以没有必要调用adapt()在这种情况下。 print(output_data.shape) print("min:", np.min(output_data)) #使用Keras Functional API构建模型 dense = keras.layers.Dense(units=16) #它将其输入映射到16维特征空间: #但是用于任何大小的RGB图像的输入将具有shape (None, None, 3)。
inputWords = [[gloveDict.get(w, [0] * dims) for w in t] for t in inputWords] #vectorise words: if the word is unique to MTG it just gets set to 0 print("Vectorised oracle text") #split data into training and test datasets testNames = [data[i]["name"] for i in range(len(data)) if data[i]["set"] == "m20"] trainVecs = [inputVecs[i] for i in range(len(inputVecs)) if not data[i]["set"] == "m20"] testVecs = [inputVecs[i] for i in range(len(inputVecs)) if data[i]["set"] == "m20"] trainWords = tf.ragged.constant([inputWords[i] for i in range(len(inputWords)) if not data[i]["set"] == "m20"]) testWords = tf.ragged.constant([inputWords[i] for i in range(len(inputWords)) if data[i]["set"] == "m20"]) trainCorRars = [corRars[i] for i in range(len(corRars)) if not data[i]["set"] == "m20"] testCorRars = [corRars[i] for i in range(len(corRars)) if data[i]["set"] == "m20"] #normalise input vectors trainVecs = np.array(trainVecs).astype("float32") #convert to numpy format normalizer = Normalization() normalizer.adapt(trainVecs) trainVecs = normalizer(trainVecs) testVecs = normalizer(testVecs) print("Normalised numerical data") #build keras model wordIn = layers.Input(shape = (None, len(inputWords[0][0]))) #input layer for var-length word vec data numIn = layers.Input(shape = (len(inputVecs[0]), )) #input layer for fixed-length numerical data rnn = layers.LSTM(32)(wordIn) #RNN layer for word vec data numLayer = layers.Dense(20, activation = "relu")(numIn) #layer for numerical data merge = layers.concatenate([rnn, numLayer]) #combine the two vectors combLayer = layers.Dense(64, activation = "relu")(merge) #hidden intermediate layer for combined data out = layers.Dense(3, activation = "softmax")(combLayer) #final layer: softmax ensures output is a set of probabilities model = DualModel(inputs = [numIn, wordIn], outputs = [out]) model.compile(loss = "sparse_categorical_crossentropy", metrics = "sparse_categorical_accuracy", optimizer = "adam") #I have no idea whether these ones are the best ones to use print("Built model")