def preprocessing_data_mnist(x_dev, y_dev, x_test, y_test): print("x_dev : ", x_dev.shape, "\n") #(60000, 28, 28) print("y_dev : ", y_dev.shape, "\n") #(60000,) print("x_test : ", x_test.shape, "\n") #(10000, 28, 28) print("y_test : ", y_test.shape, "\n") #(10000,) #Preprocessing simple : normalisation, on ferait bien plus si les images avaient des dimensions différentes !! #Normalisation : mettre à échelle de -1 et 1 selon les extremum de l'array concernée x_dev_n = utils.normalize(x_dev, axis=1) x_test_n = utils.normalize(x_test, axis=1) #Creation du set de validation : 20% du dev set x_train_n, x_val_n, y_train, y_val = train_test_split(x_dev_n, y_dev, test_size=0.2, random_state=1) #Reshape des features pr avoir bonne dimensions, -1 car infini mais on pourrait mettre len(x_train)... x_train_n_r = x_train_n.reshape(-1, 28, 28, 1) x_val_n_r = x_val_n.reshape(-1, 28, 28, 1) x_test_n_r = x_test_n.reshape(-1, 28, 28, 1) #OnHotEncoder sur label y_train_c = to_categorical(y_train, num_classes=10) y_val_c = to_categorical(y_val, num_classes=10) y_test_c = to_categorical(y_test, num_classes=10) #convert in array (mandatory with tensorflow) y_train_a = np.array(y_train_c) y_val_a = np.array(y_val_c) y_test_a = np.array(y_test_c) return (x_train_n_r, x_test_n_r, x_val_n_r), (y_train_a, y_test_a, y_val_a)
def prepare_input(self): resources = self.get_resources() upgrades = self.get_upgrades() in_progress = self.get_units_in_progress() friendly_unit_list = self.get_friendly_unit_list() enemy_unit_list = self.get_enemy_unit_list() input_data = [] input_data.append(self.observation.game_loop) # 1 input_data += resources # 9 input_data += upgrades # 26 input_data += in_progress # 70 input_data += friendly_unit_list # 44 input_data += enemy_unit_list # 44 # print('Time step (step/seconds): ' + str(self.observation.game_loop) + '/' + str(self.observation.game_loop/22.4)) # print('Resources (minerals, vespene, food(cap, used, army, workers), idle_workers, army_count, warp_gates): ' + str(resources[0]) + ', ' + str(resources[1]) + ', (' + str(resources[2]) + ', ' + str(resources[3]) + ', ' + str(resources[4]) + ', ' + str(resources[5]) + '), ' + str(resources[6]) + ', ' + str(resources[7]) + ', ' + str(resources[8])) # print('In progress: ' + str(self.in_progress_dic(in_progress))) # print('Upgrades: ' + str(self.upgrades_dic(upgrades))) # print('Friendly buildings: ' + str(self.buildings_dic(friendly_unit_list))) # print('Friendly units: ' + str(self.units_dic(friendly_unit_list))) # print('Enemy buildings: ' + str(self.buildings_dic(enemy_unit_list))) # print('Enemy units: ' + str(self.units_dic(enemy_unit_list))) input_data = normalize(input_data, axis=-1, order=2) # input_data = min_max_norm(input_data, self.maxes) return np.array(input_data)
def __getitem__(self, idx): # Use ordering of self.indexes for dataset inds = self.indexes[idx * self.batch_size:(idx + 1) * self.batch_size] batch_x, batch_y = [], [] for i in inds: # Load data from .gz data = np.loadtxt(self.filenames[i]) # Check and if necessary correct data length diff = len(data) - 12000 if (diff < 0): data = np.pad(data, (0, abs(diff)), mode="constant") elif (diff > 0): data = data[:-diff] # Handles the shape of the chosen input data, adding a filter dimension. input_types = { TIME_SEQUENCE: lambda data: data.reshape(12000, 1), LOG_SPECTROGRAM: lambda data: clipped_log_spectrogram(data).reshape(129, 53, 1), LINEAR_SPECTROGRAM: lambda data: normalize( get_spectrogram(data).reshape(129, 53, 1)) } # Add data and label to batch arrays batch_x.append(input_types[self.input_type](data)) batch_y.append(self.labels[i]) output = np.array(batch_x), np.array(list(map(self.get_ys, batch_y))) return output
def obtain_dataset_wordvectors(dataset_file="", labels_file="", sequence_file="", maxlen=1500): dataset = [] sequences = [] lengths = [] for ix, i in tqdm(enumerate(open(dataset_file))): i = i.split() item = np.array([float(k) for k in i]) dataset.append(item) lengths.append(len(item)) for i in tqdm(open(sequence_file)): item = [[aa2int(k)] for k in i] sequences.append(item) sequences = pad_sequences(sequences, maxlen=maxlen, padding="post", dtype="float32", truncating="post") dataset = np.array(dataset) sequences = np.array(sequences) print(dataset.shape, sequences.shape, set(lengths)) return normalize(dataset, axis=-1, order=2), sequences
def classifyData(featuresList, model): npFeaturesList = np.array(featuresList, dtype='float64') normalizedFeaturesList = normalize(npFeaturesList, axis=1) predictionVect = model.predict(normalizedFeaturesList) prediction = int(np.round(np.average(np.argmax(predictionVect, axis=0)))) return prediction
def neural_network_sklearn(x_train, x_test, y_train, y_test): tf_x_train = np.asarray(x_train[:]) tf_y_train = np.asarray(y_train[:]) tf_x_test = np.asarray(x_test[:]) tf_y_test = np.asarray(y_test[:]) tf_x_train = normalize(tf_x_train, axis=1) tf_x_test = normalize(tf_x_test, axis=1) regr = MLPRegressor(random_state=1, max_iter=50000, learning_rate='adaptive').fit(tf_x_train, tf_y_train) regr.predict(tf_x_test[:2]) score = regr.score(tf_x_test, tf_y_test) print("Neural Network Sklearn") print(score)
def prepareForModel(input): """Normalizes and resizes `input` retaining number of inputs resizes = n_inputs x `input.shape[1]` x `input.shape[2]` x 1 """ input = utils.normalize(input, axis=1) input = input.reshape((len(input), input.shape[1], input.shape[2], 1)) return input
def down_and_normal(x, d): """ Function for downsampling the data, normalizing and improving numerical stability. """ x = dec(x, d) #downsampling fn = lambda a: a * 1e6 # improves numerical stability x = fn(x) x = normalize(x) return x.astype(np.float32)
def __init__(self, train_x, test_x, train_y, test_y, classes): self.train_x_, self.test_x_ = normalize(train_x), normalize(test_x) train_x_pca_, test_x_pca_ = principle_components(self.train_x_, self.test_x_) self.train_x_pca_, self.test_x_pca_ = normalize(train_x_pca_), normalize(test_x_pca_) self.classes = classes print(train_y[0], train_y[2], train_y[len(train_y) - 2]) def neg_1(x): new_ = [0.0 if i == 2.0 else i for i in x] new_2 = [1.0 if j == 3.0 else j for j in new_] return new_2 train_y, test_y = list(neg_1(train_y)), list(neg_1(test_y)) # train_y, test_y = list(train_y), list(test_y) print(train_y[0], train_y[2], train_y[len(train_y) - 2]) self.train_y_cat, self.test_y_cat = to_categorical(train_y), to_categorical(test_y) print(self.train_y_cat[0], self.train_y_cat[2], self.train_y_cat[len(train_y) - 2])
def run_predict_isAtk(df): ''' Separate attacks traffic and normal traffic Parameters ---------- df : DataFrame Return ------ [Success] output : dict Normal traffic df2 : DataFrame Attack traffic to be analyse ''' output = {} protocols = { num: name[8:] for name, num in vars(socket).items() if name.startswith("IPPROTO") } columnList = df.columns df1 = df.copy() logID = df.pop('ID') sourceIP = df.pop('SourceIP') protocol = df['Protocol'].values port_num = df['Dst Port'].values time = df[['Timestamp']].values df_test = normalize(df.values) model = load_model("modules/model/binary_class_classifier.h5") model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) predictions = model.predict_proba(df_test) df2 = DataFrame(columns=columnList) for p, q, r, s, t, sip in zip(predictions, logID, protocol, port_num, time, sourceIP): if argmax(p) == 0: output[int(q)] = { "IsAtk": argmax(p), "IP": sip, "Protocol": protocols[r], "Port": int(s), "Time": int(t[0]) } else: row_index = df1.loc[df1["ID"] == q].index[0] df2.loc[df1.index[row_index]] = df1.iloc[row_index] return output, df2
def image_transormation(self, filename): img = Image.open(filename) size = (28, 28) img = img.resize(size) img.save(filename) img_array = cv2.imread(filename, cv2.IMREAD_GRAYSCALE) img_array = cv2.bitwise_not(img_array) img_array_norm = utils.normalize(img_array, axis=1) return img_array_norm
def predict_result_extern_data(model, image, label): label_true = label image = cv2.resize(image, (28, 28), interpolation=cv2.INTER_AREA) #image = np.dot(image[...,:3], [0.299, 0.587, 0.144]) image = image.reshape(1, 28, 28, 1) image = utils.normalize(image, axis=1) image = np.array(image) predict = model.predict(image) #label_pred = np.argmax(model.predict(y_test_a[index_test_set])) label_pred = np.argmax(predict) print("Data extern label : ", label_true) print("Modèle prédit : ", (predict), "donc : ", label_pred) plt.imshow(image) plt.show()
def down_and_normal(self, data1=True, data2=False, downsample_rate1=2, downsample_rate2=2, norm=True): """ Downsample and normalize the data. :params: data1 (bool): apply to data1 or not :params: data2 (bool): apply to data2 or not :params: downsample_rate1 (int): downsample rate. :params: downsample_rate2 (int): downsample rate. :params: norm: (bool) to normalize or not to normalize. Returns: n_trial * n_chans * n_samples Numpy array containing downsampled and/or normalized data. """ if data1 == False and data2 == False: raise ValueError(f"Require at least one data type to be True: data1:{data1}, data2:{data2}") else: fnc = lambda a: a * 1e6 # improves numerical stability if data1: self.downsample_rate1 = downsample_rate1 if self.downsample_rate1 > 1: self.data1 = dec(self.data1, downsample_rate1) self.downsampled[0] = True self.data1 = fnc(self.data1) if norm: self.data1 = normalize(self.data1) self.normalized[0] = True if data2: self.downsample_rate2 = downsample_rate2 if self.downsample_rate2 > 1: self.data2 = dec(self.data2, downsample_rate2) self.downsampled[1] = True self.data2 = fnc(self.data2) if norm: self.data2 = normalize(self.data2) self.normalized[1] = True
def __init__(self, target_shape, offset=None, data_format=None, **kwargs): ''' Crop to target. If only one `offset` is set, then all dimensions are offset by this amount. ''' super(CroppingLike2D, self).__init__(**kwargs) self.data_format = normalize(data_format) self.target_shape = target_shape if offset is None or offset == 'centered': self.offset = 'centered' elif isinstance(offset, int): self.offset = (offset, offset) elif hasattr(offset, '__len__'): if len(offset) != 2: raise ValueError('`offset` should have two elements. Found: ' + str(offset)) self.offset = offset self.input_spec = InputSpec(ndim=4)
def identifyImageProbabilities(image_list, conv2d=False): global MODEL_PATH, DIMENSION model = load_model(MODEL_PATH) if (not isinstance(image_list, list)): image = cv2.resize(image_list, (DIMENSION, DIMENSION), interpolation=cv2.INTER_AREA) image_list = [image] image_list_as_array = np.asarray(image_list) image_list_as_array = normalize(image_list_as_array, axis=1) if (conv2d): return model.predict( image_list_as_array.reshape(-1, DIMENSION, DIMENSION, 1)) else: return model.predict(image_list_as_array)
def get_embedding(embedding_dim, batch_size, epochs): # build model x = Input(shape=(1,)) o = Embedding(input_dim=997, output_dim=embedding_dim, embeddings_initializer=he_normal(), name='embedding')(x) h = Dense(128, use_bias=False, kernel_initializer=he_normal(), activation='relu')(o) h = Dense(24 * 5, use_bias=False, kernel_initializer=he_normal(), activation='relu')(o) model = Model(inputs=x, outputs=h) model.compile(loss='mse', optimizer=Adam(3e-4)) # train embedding weights hist = model.fit(np.arange(0, 997).reshape(-1, 1), normalize(embed_label.values), batch_size=batch_size, epochs=epochs, shuffle=True, verbose=0) # output embedding vector areaEmbedding = model.get_weights()[0] return areaEmbedding, hist
def make_triplet(alert, to_tpu: bool = False): """ Feed in alert packet """ cutout_dict = dict() for cutout in ('science', 'template', 'difference'): cutout_data = loads( dumps([alert[f'cutout{cutout.capitalize()}']['stampData']]))[0] # unzip with gzip.open(io.BytesIO(cutout_data), 'rb') as f: with fits.open(io.BytesIO(f.read())) as hdu: data = hdu[0].data # replace nans with zeros cutout_dict[cutout] = np.nan_to_num(data) # normalize cutout_dict[cutout] = normalize(cutout_dict[cutout]) # pad to 63x63 if smaller shape = cutout_dict[cutout].shape if shape != (63, 63): # print(f'Shape of {candid}/{cutout}: {shape}, padding to (63, 63)') cutout_dict[cutout] = np.pad(cutout_dict[cutout], [(0, 63 - shape[0]), (0, 63 - shape[1])], mode='constant', constant_values=1e-9) triplet = np.zeros((63, 63, 3)) triplet[:, :, 0] = cutout_dict['science'] triplet[:, :, 1] = cutout_dict['template'] triplet[:, :, 2] = cutout_dict['difference'] if to_tpu: # Edge TPUs require additional processing triplet = np.rint(triplet * 128 + 128).astype(np.uint8).flatten() return triplet
def show_results(img_ext): img_ext = cv2.resize(img_ext, (28, 28), interpolation=cv2.INTER_AREA) #image = np.dot(image[...,:3], [0.299, 0.587, 0.144]) #img_ext = img_ext.reshape(1, 28, 28, 1) img_ext = utils.normalize(img_ext, axis=1) img_ext = np.array(img_ext) plt.figure() plt.subplot(2, 3, 1) plt.title("MNIST : prediction 7 (99%)", color='blue', size=18) plt.imshow(x_test_n_r[0], cmap='plasma') plt.subplot(2, 3, 2) plt.title("MNIST : prediction 5 (95%)", color='blue', size=18) plt.imshow(x_test_n_r[52], cmap='plasma') plt.subplot(2, 3, 3) plt.title("MNIST : prediction 6 (91%)", color='blue', size=18) plt.imshow(x_test_n_r[54], cmap='plasma') plt.subplot(2, 3, 4) plt.title("MNIST : prediction 4 (96%)", color='blue', size=18) plt.imshow(x_test_n_r[56], cmap='plasma') plt.subplot(2, 3, 5) plt.title("Extern data : prediction 8 (95%)", color='blue', size=18) plt.imshow(img_ext, cmap='plasma') plt.show()
def get(self, split='train', preprocess='line_profile', categorical_labels=True): x, y = self.load_data(split) if preprocess == 'line_profile': x = self.get_line_profiles(x, x.shape[1]) elif preprocess == 'flattened': x = x.reshape((x.shape[0], -1)) elif preprocess == 'images': n, h, w = x.shape x = x.reshape((n, h, w, 1)) x = x.astype('float32') x /= 255.0 if categorical_labels: num_classes = len(np.unique(y)) y = to_categorical(y, num_classes) x = normalize(x) return x, y
if (image_name.split('.')[1] == 'jpg'): image = cv2.imread(test_image_directory + image_name, 0) image = Image.fromarray(image) image = image.resize((SIZE, SIZE)) X_test.append(np.array(image)) masks = os.listdir(test_mask_directory) for i, mask_name in enumerate(masks): if (mask_name.split('.')[1] == 'jpg'): mask = cv2.imread(test_mask_directory + mask_name, 0) mask = Image.fromarray(mask) mask = mask.resize((SIZE, SIZE)) Y_test.append(np.array(mask)) X_test = np.expand_dims(X_test, axis=3) X_test = normalize(X_test, axis=1) Y_test = np.expand_dims( np.array(Y_test), axis=3) / 255. #change the 255 if masks already normalized test_masks_cat = to_categorical(Y_test, num_classes=n_classes) y_test_cat = test_masks_cat.reshape( (Y_test.shape[0], Y_test.shape[1], Y_test.shape[2], n_classes)) #check this line x_train = np.expand_dims(x_train, axis=3) x_train = normalize(x_train, axis=1) y_train = np.expand_dims( np.array(y_train), axis=3) / 255. #change the 255 if masks already normalized
# In[8]: f_mnist = fashion_mnist # In[59]: (x_tr, y_tr), (x_te, y_te) = f_mnist.load_data() # In[60]: x_tr, x_te = normalize(x_tr), normalize(x_te) # In[61]: x_tr.shape # In[62]: x_tr[0].shape # In[63]:
def __init__(self, scale_factor=2, data_format=None, **kwargs): super(SubPixelUpscaling, self).__init__(**kwargs) self.scale_factor = scale_factor self.data_format = normalize(data_format)
def load_data_part_of_game( data_path, train, validation, test, time_start, time_end, maxes_path=None, seed=None, ): print('Loading data...') # Make list of the paths to all the replays data_paths = [] for file in os.listdir(data_path): file_path = os.path.join(data_path, file) if os.path.isfile(file_path) and file.lower().endswith('.npy'): data_paths.append(file_path) if seed is not None: np.random.seed(seed) np.random.shuffle(data_paths) train_end = int(len(data_paths) * train) validation_end = int(len(data_paths) * (train + validation)) train_paths = [] for index in range(train_end): train_paths.append(data_paths[index]) validation_paths = [] for index in range(train_end, validation_end): validation_paths.append(data_paths[index]) amount_train_data_points = 0 for path in train_paths: for data_point in np.load(path): if len(data_point ) == 248 and time_start <= data_point[0] < time_end: amount_train_data_points += 1 amount_validation_data_points = 0 for path in validation_paths: for data_point in np.load(path): if len(data_point ) == 248 and time_start <= data_point[0] < time_end: amount_validation_data_points += 1 data = [] labels = [] for path in data_paths: for data_point in np.load(path): if len(data_point ) == 248 and time_start <= data_point[0] < time_end: data.append(data_point[:-54]) labels.append(data_point[-54:]) print('Data loaded.') if len(data) == 0: return np.array([]), np.array([]), np.array([]), np.array( []), np.array([]), np.array([]), print('Performing L2 normalization...') data = normalize(data, axis=-1, order=2) print('L2 normalization done.') # print('Performing min-max normalization...') # min_max_norm(data, maxes_path) # print('Min-max normalization done.') print('Splitting data...') train_end = amount_train_data_points validation_end = amount_train_data_points + amount_validation_data_points train_data = [] train_labels = [] for index in range(train_end): train_data.append(data[index]) train_labels.append(labels[index]) validation_data = [] validation_labels = [] for index in range(train_end, validation_end): validation_data.append(data[index]) validation_labels.append(labels[index]) test_data = [] test_labels = [] for index in range(validation_end, len(data)): test_data.append(data[index]) test_labels.append(labels[index]) print('Data split.') print( '_____________________________________________________________________________________' ) print('Data meta data') print('{:20s} {:7d}'.format('# of games', len(data_paths))) print('{:20s} {:7d}'.format('# of data points', len(data))) print('Split seed: ' + str(seed)) print( '-------------------------------------------------------------------------------------' ) print('| {:25s} | {:25s} | {:25s} |'.format('Data', '# data points', '# data point dimensions')) print( '|---------------------------|---------------------------|---------------------------|' ) print('| {:25s} | {:25d} | {:25d} |'.format('train_data shape', len(train_data), len(train_data[0]))) print('| {:25s} | {:25d} | {:25d} |'.format('train_labels shape', len(train_labels), len(train_labels[0]))) print('| {:25s} | {:25d} | {:25d} |'.format('validation_data shape', len(validation_data), len(validation_data[0]))) print('| {:25s} | {:25d} | {:25d} |'.format('validation_labels shape', len(validation_labels), len(validation_labels[0]))) print('| {:25s} | {:25d} | {:25d} |'.format('test_data shape', len(test_data), len(test_data[0]))) print('| {:25s} | {:25d} | {:25d} |'.format('test_labels shape', len(test_labels), len(test_labels[0]))) print( '-------------------------------------------------------------------------------------' ) return np.array(train_data), np.array(train_labels), np.array( validation_data), np.array(validation_labels), np.array( test_data), np.array(test_labels)
from tensorflow.keras.utils import normalize import tensorflow.keras.datasets.mnist as mnist import numpy as np (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train, x_test = normalize(x_train), normalize(x_test) x_train_2D, x_test_2D = x_train[:, np.newaxis, ...], x_test[:, np.newaxis, ...] x_train = np.reshape(x_train, (60000, 28 * 28)) y_train_reformatted = np.zeros((60000, 10)) for i, correct in enumerate(y_train): y_train_reformatted[i, correct] = 1 x_test = np.reshape(x_test, (10000, 28 * 28)) y_test_reformatted = np.zeros((10000, 10)) for i, correct in enumerate(y_test): y_test_reformatted[i, correct] = 1
from tensorflow.keras.callbacks import TensorBoard import numpy as np import tensorflow as tf import pickle import cv2 import time # Load the dataset X = pickle.load(open("X.pickle", "rb")) Y = pickle.load(open("Y.pickle", "rb")) modelname = "a-z{}".format(int(time.time())) #board = TensorBoard(Log_dir="logs/{}".format(modelname)) # Scaling the data. /255 since data is image data X = normalize(X, axis=1) print("({})".format(X.shape[0] / 2400) + str(X.shape)) # (2400,50,50,1) - n,y,x,c model = Sequential() model.add(Conv2D(64, (3, 3), input_shape=X.shape[1:])) # 64 3,3 model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) # model.add(Dropout(0.40)) model.add(Conv2D(128, (3, 3))) # 5,5 model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) # EXTRA model.add(Conv2D(64, (3, 3))) # 5,5
def run_predict_Atks(df): ''' Identify Possible Attacks Parameters ---------- df : DataFrame Return ------ output : dict Labels Attack Traffic ''' columnList = df.columns df1 = df.copy() output = {} protocols = { num: name[8:] for name, num in vars(socket).items() if name.startswith("IPPROTO") } atks = { 0: 'Bot', 1: 'Brute Force -Web', 2: 'Brute Force -XSS', 3: 'DDOS attack-HOIC', 4: 'DDOS attack-LOIC-UDP', 5: 'DoS attacks-GoldenEye', 6: 'DoS attacks-Hulk', 7: 'DoS attacks-SlowHTTPTest', 8: 'DoS attacks-Slowloris', 9: 'FTP-BruteForce', 10: 'Infiltration', 11: 'SQL Injection', 12: 'SSH-Bruteforce' } logID = df.pop('ID') SourceIP = df.pop('SourceIP') time = df[['Timestamp']].values protocol = df[['Protocol']].values port = df[['Dst Port']].values df_test = normalize(df.values) model = load_model( "modules/model/Atk_multiclass_categorical_50_ep_80_bs.h5") model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) predictions = model.predict_proba(df_test) for l, sip, p1, p2, p3, t in zip(logID, SourceIP, predictions, protocol, port, time): output[int(l)] = { "IsAtk": 1, "IP": sip, "Protocol": protocols[int(p2)], "Port": int(p3), "Atk": atks[argmax(p1)], "Time": int(t[0]) } return output
def clipped_log_spectrogram(data): Sxx = get_spectrogram(data) Sxx[Sxx == 0] = 1 min_Sxx = np.amin(Sxx) Sxx[Sxx == 1] = min_Sxx return normalize(np.log10(Sxx))
import numpy as np X = np.array(X) y = np.array(y) X_val = np.array(X_val) y_val = np.array(y_val) import tensorflow as tf from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from tensorflow.keras.utils import normalize, to_categorical from tensorflow.keras.optimizers import Adam X = normalize(X) X_val = normalize(X_val) y = normalize(y) y_val = normalize(y_val) # print(y_val.shape) # print(y.shape) model = Sequential() model.add(Dense(128, activation = 'relu')) model.add(Dense(128, activation = 'relu')) model.add(Dense(y.shape[1], activation = 'sigmoid')) opt = Adam(lr = 0.001)
epochs = range(len(loss)) plt.figure() plt.plot(epochs, loss, 'b', label='Training loss') plt.plot(epochs, val_loss, 'r', label='Validation loss') plt.title(title) plt.legend() plt.show() first = np.array(first).reshape(228, params.Fd, 4) second = np.array(second) first = normalize(first) second /= 1000 model = Sequential() model.add(LSTM(params.inputUnits, return_sequences=True, input_shape=(params.Fd, 4))) model.add(LSTM(params.hideUnits, activation=params.activationFuncInHideLayer)) model.add(Dense(params.Fh, activation=params.activationFuncInOutputLayer)) model.compile(loss=params.funcError, optimizer=RMSprop(), metrics=['mae']) history = model.fit(first, second, epochs=params.epoch, batch_size=params.batchSize, validation_split=params.validationSize) plot_train_history(history, "График") print("Пожалуйста, подождите")
# ============================================================= # plt.imshow(TRAIN_IMAGS[24], cmap="gray") # plt.show() # plt.imshow(TRAIN_MASKS[24]) # plt.show() # Assign labels & encode them. labeler = LabelEncoder() n, h, w = TRAIN_MASKS.shape reshaped_masks = TRAIN_MASKS.reshape(-1, 1) encoded_masks = labeler.fit_transform(reshaped_masks) updated_masks = encoded_masks.reshape(n, h, w) # Prepare training datasets. TRAIN_IMAGS = np.expand_dims(TRAIN_IMAGS, axis=3) TRAIN_IMAGS = normalize(TRAIN_IMAGS, axis=1) # NOTE: Normalization step input_masks = np.expand_dims(updated_masks, axis=3) # Create training & testing datasets. N_TEST = 0.1 x_train, x_test, y_train, y_test = train_test_split(TRAIN_IMAGS, input_masks, test_size=N_TEST, random_state=0) # NOTE: Sanity check # print("Class values in the dataset are ... ", np.unique(y_train)) # Categorize by one-hot encoding. masks_cat_train = to_categorical(y_train, num_classes=N_CLASSES) y_train_cat = masks_cat_train.reshape(