'PetalLength',
'PetalWidth',
'Species'
]
# Import training dataset
training_dataset = pd.read_csv('iris_training.csv', names=COLUMN_NAMES, header=0)
train_x = training_dataset.iloc[:, 0:4].values
train_y = training_dataset.iloc[:, 4].values
# Import testing dataset
test_dataset = pd.read_csv('iris_test.csv', names=COLUMN_NAMES, header=0)
test_x = test_dataset.iloc[:, 0:4].values
test_y = test_dataset.iloc[:, 4].values
# Encoding training dataset
encoding_train_y = to_categorical(train_y)
# Encoding testing dataset
encoding_test_y = to_categorical(test_y)
#print(encoding_train_y)
# Creating a model
model = Sequential()
model.add(Dense(10, input_dim=4, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(3, activation='softmax'))
# Compiling model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
d[sequence_length * i:sequence_length * (i + 1)]
for i in range(d.shape[0] // sequence_length)
] + [d[-sequence_length:]] for d in train_data]
train_data = np.array([_ for d in train_data for _ in d])
valid_data = [
_valid_data[l:r] for i in range(valid_subjects)
for l, r in (sample_indexes + sample_per_subject * i)
]
valid_data = [[
d[sequence_length * i:sequence_length * (i + 1)]
for i in range(d.shape[0] // sequence_length)
] + [d[-sequence_length:]] for d in valid_data]
valid_data = np.array([_ for d in valid_data for _ in d])
train_X, train_y = train_data[..., :310], to_categorical(
train_data[:, 0, 310], num_classes=labels_num)
valid_X, valid_y = valid_data[..., :310], to_categorical(
valid_data[:, 0, 310], num_classes=labels_num)
print(train_X.shape, train_y.shape, valid_X.shape, valid_y.shape)
def ModelLSTM():
model = Sequential()
model.add(
LSTM(units=128,
input_shape=(sequence_length, 310),
activation='tanh',
return_sequences=False))
model.add(Dense(units=labels_num, activation='softmax'))
return model
def __init__(self,
folds,
img_height,
img_width,
height_scaled,
width_scaled,
mean,
std,
batch_size=64,
shuffle=True):
df = pd.read_csv("../input/train_folds.csv")
# print(df.shape)
df = df[[
"image_id", "grapheme_root", "vowel_diacritic",
"consonant_diacritic", "kfold"
]]
df = df[df.kfold.isin(folds)].reset_index(drop=True)
self.image_ids = df.image_id.values
self.grapheme_root = to_categorical(df.grapheme_root.values)
self.vowel_diacritic = to_categorical(df.vowel_diacritic.values)
self.consonant_diacritic = to_categorical(
df.consonant_diacritic.values)
self.batch_size = batch_size
self.shuffle = shuffle
self.on_epoch_end()
if (len(folds) == 1):
self.aug = albumentations.Compose([
albumentations.Resize(height_scaled,
width_scaled,
always_apply=True),
albumentations.Normalize(mean, std, always_apply=True)
])
else:
self.aug = albumentations.Compose([
albumentations.Resize(height_scaled,
width_scaled,
always_apply=True),
# RandomRotate90(),
# Flip(),
# Transpose(),
# OneOf([
# IAAAdditiveGaussianNoise(),
# GaussNoise(),
# ], p=0.2),
# OneOf([
# MotionBlur(p=0.2),
# MedianBlur(blur_limit=3, p=0.1),
# Blur(blur_limit=3, p=0.1),
# ], p=0.2),
# ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.2),
# OneOf([
# OpticalDistortion(p=0.3),
# GridDistortion(p=0.1),
# IAAPiecewiseAffine(p=0.3),
# ], p=0.2),
# OneOf([
# # CLAHE(clip_limit=2),
# IAASharpen(),
# IAAEmboss(),
# RandomBrightnessContrast(),]),
albumentations.Cutout(num_holes=8,
max_h_size=40,
max_w_size=20,
fill_value=0,
always_apply=False,
p=0.5),
CoarseDropout(max_holes=8, max_height=40, max_width=20, p=0.5),
albumentations.Normalize(mean, std, always_apply=True),
# HorizontalFlip(),
])
from tensorflow.keras import datasets, utils
from tensorflow.keras import layers, models
from tensorflow.keras import optimizers
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
''' Prepare the data '''
(trainX, trainY), (testX, testY) = datasets.cifar10.load_data()
trainY = trainY.reshape(50000)
testY = testY.reshape(10000)
trainY = utils.to_categorical(trainY)
testY = utils.to_categorical(testY)
trainX = trainX.astype('float32')
testX = testX.astype('float32')
trainX = trainX / 255.0
testX = testX / 255.0
# Create validation data set
validX = trainX[0:5000]
validY = trainY[0:5000]
trainX = trainX[5000:]
trainY = trainY[5000:]
''' Create the NN-model '''
model = models.Sequential()
model.add(
layers.Conv2D(32, (3, 3),
#Converting lists into numpy arrays
data = np.array(data)
labels = np.array(labels)
print(data.shape, labels.shape)
#Splitting training and testing dataset
X_train, X_test, y_train, y_test = train_test_split(data,
labels,
test_size=0.2,
random_state=42)
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
#Converting the labels into one hot encoding
y_train = to_categorical(y_train, 43)
y_test = to_categorical(y_test, 43)
#Building the model
model = Sequential()
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
activation='relu',
input_shape=X_train.shape[1:]))
model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(rate=0.25))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
s = np.arange(Cells.shape[0])
np.random.seed(num_classes)
np.random.shuffle(s)
Cells = Cells[s]
labels = labels[s]
#Data Augmentation
(Xnew_train,
Xnew_val) = Cells[(int)(0.2 * len(labels)):], Cells[:(int)(0.2 * len(labels))]
Xnew_train = Xnew_train.astype('float32') / 255
Xnew_val = Xnew_val.astype('float32') / 255
(ynew_train,
ynew_val) = labels[(int)(0.2 * len(labels)):], labels[:(int)(0.2 *
len(labels))]
ynew_train = to_categorical(ynew_train, num_classes)
ynew_val = to_categorical(ynew_val, num_classes)
epochs = 1
#Comparing accuracy of the model on two different datasets
history1 = model.fit(Xnew_train,
ynew_train,
batch_size=32,
epochs=epochs,
validation_data=(Xnew_val, ynew_val))
history = model.fit(X_train,
y_train,
batch_size=32,
epochs=epochs,
from tensorflow.keras.utils import to_categorical
max_len = 50
X = [[word2idx[w[0]] for w in s] for s in sentences]
X = pad_sequences(maxlen=max_len,
sequences=X,
padding='post',
value=num_words - 1)
y = [[tag2idx[w[2]] for w in s] for s in sentences]
y = pad_sequences(maxlen=max_len,
sequences=y,
padding='post',
value=tag2idx["O"])
y = [to_categorical(i, num_classes=num_tags) for i in y]
# Building BILSTM
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.1,
random_state=1)
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import LSTM, Embedding, Dense
from tensorflow.keras.layers import TimeDistributed, SpatialDropout1D, Bidirectional
input_word = Input(shape=(max_len, ))
]
def grayscale(data):
data_holder = []
for i in range(len(data)):
data_holder.append(rgb2gray(data[i]))
data_holder = np.array(data_holder)
data_holder = data_holder.reshape(len(data_holder), 32, 32, 1)
return data_holder
# Loading data
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_test = to_categorical(y_test, 10)
# convert to grayscale
x_train = grayscale(x_train)
x_test_modif = grayscale(x_test)
model = tf.keras.models.load_model('cifar80%accuracy.model')
prediction = model.predict(x_test_modif)
plt.figure(figsize=(15, 10))
for i in range(10):
random = np.random.randint(0, 10000)
plt.subplot(5, 2, i + 1)
plt.imshow(x_test[random])
index = np.argmax(prediction[random])
plt.ylabel('{}'.format(classification_list[index]))
test_batch = (unpickle('./cifar-10-batches-py/test_batch'))
x_train = batches[0][b'data']
y_train = batches[0][b'labels']
for i in range(1, len(batches)):
x_train = np.concatenate((x_train, batches[i][b'data']), axis=0)
y_train = y_train + batches[i][b'labels']
x_test = test_batch[b'data']
y_test = test_batch[b'labels']
x_train = np.reshape(x_train, [50000, 32, 32, 3])
x_test = np.reshape(x_test, [10000, 32, 32, 3])
from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)
print("Input shape: {}".format(x_train[0].shape))
print("Training Set: {} samples".format(len(x_train)))
print("Test Set: {} samples".format(len(x_test)))
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
def clear_history(self, logs={}):
# In[7]:
# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255.
X_test /= 255.
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = to_categorical(y_train, nb_classes)
Y_test = to_categorical(y_test, nb_classes)
hidden_dim = task_params['hidden_dim']
model = Sequential()
model.add(Dense(hidden_dim, input_shape=(784, )))
model.add(Activation('relu'))
# model.add(Dropout(0.2))
model.add(Dense(hidden_dim))
model.add(Activation('relu'))
# model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
model.summary()
y = patients.take([13], axis=1)
# Normalize X data
colT = ColumnTransformer([
("onehot",
OneHotEncoder(
categories=[[0, 1], [1, 2, 3, 4], [0, 1], [0, 1, 2], [0, 1],
[1, 2, 3], [0, 1, 2, 3], [3, 6, 7]]),
[1, 2, 5, 6, 8, 10, 11, 12]),
("norm", Normalizer(norm='l1'), [0, 3, 4, 7, 9])
])
x = colT.fit_transform(x)
# Normalize Y data
y = to_categorical(y)
# Split the data into train and test data
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=0)
# Determined parameters
n_input = x_train.shape[1]
n_output = y_train.shape[1]
input_layer = layers.Input(shape=(n_input, ))
hidden_layers = layers.Dense(hidden_nodes,
activation=activations.elu)(input_layer)
output_layer = layers.Dense(n_output,
activation=activations.sigmoid)(hidden_layers)
model = Model(input_layer, output_layer)
return batch_loss
if __name__ == '__main__':
print("preparing dataset...")
t = tsp.Tsp()
X, Y = t.next_batch(100) #10000
#X.shape
#(100, 10, 2)
#Y.shape
#(100, 10)
x_test, y_test = t.next_batch(2)
YY = []
for y in Y:
YY.append(to_categorical(y))
YY = np.asarray(YY)
hidden_size = 128
seq_len = 10
nb_epochs = 100
learning_rate = 0.1
# print("building model...")
# main_input = Input(shape=(seq_len, 2), name='main_input')
# ##返回返回的是一个list,[encoder,state_h, state_c],其中encoder.shape=(batch_size,time_step,dim),state_h.shape=state_c.shape=(batch_size,dim)
# encoder,state_h, state_c = LSTM(hidden_size,return_sequences = True, name="encoder",return_state=True)(main_input)
# decoder = PointerLSTM(hidden_size, name="decoder")(encoder,states=[state_h, state_c])
#
# model = Model(main_input, decoder)
#
def do_cnn_1d(trainX, testX, trainY, testY, vocab):
'''
model = text cnn
Args:
trainX:
testX:
trainY:
testY:
Returns:
'''
# Converting labels to binary vectors
trainY = to_categorical(trainY, num_classes=4)
testY = to_categorical(testY, num_classes=4)
# Building convolutional network
input_data = Input(shape=(1000, )) # (maxlen, )
embedding = Embedding(input_dim=len(vocab) + 1, output_dim=64)(input_data)
cnn1 = Conv1D(filters=32, kernel_size=3, strides=1,
padding='valid')(embedding)
cnn1 = MaxPool1D(4, padding='valid')(cnn1)
# cnn2 = Conv1D(filters=32, kernel_size=4, strides=1, padding='valid')(embedding)
# cnn2 = MaxPool1D(4, padding='valid')(cnn2)
# cnn3 = Conv1D(filters=32, kernel_size=5, strides=1, padding='valid')(embedding)
# cnn3 = MaxPool1D(4, padding='valid')(cnn3)
# cnn = Concatenate(axis=-1)([cnn1, cnn2, cnn3])
flatten = Flatten()(cnn1)
dropout = Dropout(0.2)(flatten)
# batchNormalization = BatchNormalization()(dropout1)
# dense1 = Dense(16, activation='relu')(batchNormalization)
# dropout2 = Dropout(0.2)(dense1)
output_data = Dense(4, activation='softmax')(dropout)
model = Model(inputs=[input_data], outputs=[output_data])
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=0.0001),
metrics=['accuracy'])
model.summary()
model.fit(trainX,
trainY,
validation_data=(testX, testY),
epochs=10,
batch_size=128,
verbose=1)
# y_pred = model.predict(x_test)
# y_pred_idx = [1 if prob[0] > 0.5 else 0 for prob in y_pred]
# print(f1_score(y_test, y_pred_idx))
# print(confusion_matrix(y_test, y_pred_idx))
# 评估误差和准确率
# result = model.predict(testX) # 预测的是类别，结果就是类别号
# result_labels = np.argmax(result, axis=1)
# y_predict = list(map(str, result_labels))
# print('准确率', metrics.accuracy_score(testY, result_labels))
# print('平均f1-score:', metrics.f1_score(testY, result_labels, average='weighted'))
score, acc = model.evaluate(testX, testY, batch_size=10)
print('Test score:', score)
print('Test accuracy:', acc)
train_text = train_text + " ".join(n['patterns']) + " ".join(
n['responses'])
#Creacion del vocabulario
corpus = set(text_to_word_sequence(train_text))
corpus = list(sorted(corpus))
df = pd.DataFrame({'words': corpus})
df.to_csv('corpus.csv')
size = len(corpus) + 1
# Obtenemos los datos de entrada para entrenamiento
tok = Tokenizer()
tok.fit_on_texts(corpus)
tokens = tok.texts_to_sequences(p)
x_train = [np.add.reduce(to_categorical(X, size)) for X in tokens]
x_train = np.array(x_train)
# Obtenemos los datos de salida para el entrenamiento
clases = list(sorted(set(t)))
size_s = len(clases) + 1
tok2 = Tokenizer()
tok2.fit_on_texts(clases)
tokens_s = tok2.texts_to_sequences(t)
y_train = [np.add.reduce(to_categorical(Y, size_s)) for Y in tokens_s]
y_train = np.array(y_train)
model = keras.Sequential()
model.add(keras.Input(shape=x_train[0].shape))
# prepare the dataset of input to output pairs encoded as integers
seq_length = 1
dataX = []
dataY = []
for i in range(0, len(alphabet) - seq_length, 1):
seq_in = alphabet[i:i + seq_length]
seq_out = alphabet[i + seq_length]
dataX.append([char_to_int[char] for char in seq_in])
dataY.append(char_to_int[seq_out])
print(seq_in, '->', seq_out)
# reshape X to be [samples, time steps, features]
X = np.reshape(dataX, (len(dataX), seq_length, 1))
# normalize
X = X / float(len(alphabet))
# one hot encode the output variable
y = to_categorical(dataY)
# create and fit the model
batch_size = 1
model = Sequential()
model.add(
LSTM(50,
batch_input_shape=(batch_size, X.shape[1], X.shape[2]),
stateful=True))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
for i in range(300):
model.fit(X, y, epochs=1, batch_size=batch_size, verbose=2, shuffle=False)
model.reset_states()
# summarize performance of the model
### Neural Network: Setup and Run Tests ###
###########################################
#Load MNIST Data
(train_data, train_labels), (test_data, test_labels) = mnist.load_data()
# Reshape training and test data to add an additional dimension of 1 channel
train_data = train_data.reshape((60000, 28, 28, 1))
test_data = test_data.reshape((10000, 28, 28, 1))
# Revise pixel data to 0.0 to 1.0, 32-bit float
train_data = train_data.astype('float32') / 255 # ndarray/scalar op
test_data = test_data.astype('float32') / 255
# Turn 1-value labels to 10-value vectors
train_labels = to_categorical(train_labels)
test_labels = to_categorical(test_labels)
#Setup Image Generation
datagen = ImageDataGenerator(rotation_range=5,
shear_range=.2,
zoom_range=.1,
width_shift_range=2,
height_shift_range=2)
#Setup Tests into LyrParam instances
allConfigurations = []
allConfigurations = setupTests()
print("----------------------------------")
print("Number of test combinations: " + str(len(allConfigurations)))
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
#======== model paramters =======
input_path = './input/'
latent_dim = 512 // 2
#======== train paramters =======
epochs = 30
batch_size = 128
weight_path = './weight/bidirectional_lstm_model_weights.h5'
#================================
model_config = load_object(input_path+'model_config.pkl')
tar_vocab_size = model_config['tar_vocab_size']
tokenizers = load_object(input_path+'tokenizers.pkl')
encoder_input_seq, decoder_input_seq, decoder_target_seq = load_object(input_path+'inputs.pkl')
decoder_target_matrix = to_categorical(decoder_target_seq, tar_vocab_size)
model_config['latent_dim'] = latent_dim
enc_dec_model, enc_mode, dec_model = enc_dec_lstm(**model_config)
if not os.path.exists('./weight/'):
os.mkdir('./weight/')
try:
enc_dec_model.load_weights(weight_path)
print('load from previous model')
except:
print('train a new model')
checkpoint = ModelCheckpoint(filepath=weight_path,
])
#compiling the model; compute the loss and also determine the optimizer
model.compile(
'adam',
loss = 'categorical_crossentropy',
metrics = ['accuracy']
)
#train the model
model.fit(
train_images,
to_categorical(train_labels),
epochs = 5,
validation_data=(test_images, to_categorical(test_labels))
)
#the % accuracy is 97.3%
#cost is 0.0855
#predict an image
model.predict(test_images[:3])
data.append(image)
# Extract the labels from the path
label = image_path.split(os.path.sep)[-3]
# rename positive to smiling and negative to not smiling
label = "smiling" if label == "positives" else "not_smiling"
# print(label)
labels.append(label)
# normalize the raw pixels to the range [0,1]
data = np.array(data, dtype="float") / 255.0
labels = np.array(labels)
# Encode the labels
le = LabelEncoder()
labels = utils.to_categorical(le.fit_transform(labels), 2)
# Handle the data imbalance by computing the class weight
total_classes = labels.sum(axis=0)
# print("total classes: ", total_classes)
class_weights = total_classes.max() / total_classes
class_weights = {i: class_weights[i] for i in range(1, -1, -1)}
print("class weights: ", class_weights)
# class_weights = {
# 0: 1.,
# 1: 2.56
# }
# train-test split
# import matplotlib.pyplot as plt
import numpy as np
(train_X, train_Y), (test_X, test_Y) = cifar10.load_data()
n = 6
plt.figure(figsize=(20, 10))
# for i in range(n):
# plt.subplot(330+1+i)
# plt.imshow(train_X[i])
train_x = train_X.astype('float32')
test_X = test_X.astype('float32')
train_X = train_X / 255.0
test_X = test_X / 255.0
train_Y = to_categorical(train_Y)
test_Y = to_categorical(test_Y)
num_classes = test_Y.shape[1]
model = Sequential()
model.add(
Conv2D(32, (3, 3),
input_shape=(32, 32, 3),
padding='same',
activation='relu',
kernel_constraint=MaxNorm(3)))
model.add(Dropout(0.2))
model.add(
Conv2D(32, (3, 3),
activation='relu',
x_train = x_train.astype("float32")
x_test = x_test.astype("float32")
x_train = x_train[..., np.newaxis]
x_test = x_test[..., np.newaxis]
x_train /= 256.0
x_test /= 256.0
print(x_train.shape[0], "train samples")
print(x_test.shape[0], "test samples")
print(y_train[0:10])
y_train = to_categorical(y_train, NB_CLASSES)
y_test = to_categorical(y_test, NB_CLASSES)
x = x_in = Input(x_train.shape[1:-1] + (1, ))
x = QConv2D(32,
kernel_size=(3, 3),
kernel_quantizer=quantized_bits(4, 0, 1),
bias_quantizer=quantized_bits(4, 0, 1))(x)
x = QActivation("quantized_relu(4,0)")(x)
x = QConv2D(16,
kernel_size=(3, 3),
kernel_quantizer=quantized_bits(4, 0, 1),
bias_quantizer=quantized_bits(4, 0, 1))(x)
x = QActivation("quantized_relu(4,0)")(x)
x = QConv2D(8,
kernel_size=(3, 3),
x = BatchNormalization()(x)
x = Dense(
200,
activation='relu',
)(x)
x = Dropout(0.22)(x)
x = BatchNormalization()(x)
pred = Dense(11, activation='softmax')(x)
model = Model(inputs=seq, outputs=pred)
model.compile(loss='categorical_crossentropy', optimizer=Adam())
return model
Y_age = to_categorical(train['age'])
sub2 = np.zeros((X_test.shape[0], 11))
oof_pref2 = np.zeros((X.shape[0], 11))
score = []
count = 0
for i, (train_index, test_index) in enumerate(kfold.split(X, train['age'])):
print("FOLD | ", count + 1)
filepath2 = "age_weights_best_%d.h5" % count
checkpoint2 = ModelCheckpoint(filepath2,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
reduce_lr2 = ReduceLROnPlateau(monitor='val_loss',
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (224, 224))
# update the data and labels lists, respectively
data.append(image)
labels.append(label)
# convert the data and labels to NumPy arrays while scaling the pixel
# intensities to the range [0, 255]
data = np.array(data) / 255.0
labels = np.array(labels)
# perform one-hot encoding on the labels
lb = LabelBinarizer()
labels = lb.fit_transform(labels)
labels = to_categorical(labels)
# partition the data into training and testing splits using 80% of
# the data for training and the remaining 20% for testing
(trainX, testX, trainY, testY) = train_test_split(data,
labels,
test_size=0.20,
stratify=labels,
random_state=42)
# initialize the training data augmentation object
trainAug = ImageDataGenerator(rotation_range=15, fill_mode="nearest")
# load the VGG16 network, ensuring the head FC layer sets are left
# off
baseModel = VGG16(weights="imagenet",
# create a weight matrix for words in training docs (load word2vec word vectors)
decoder_embedding_matrix = np.zeros((num_cmd_tokens, 300))
for word, i in tokenizer.word_index.items():
embedding_vector = getVector(word)
if embedding_vector is not None:
decoder_embedding_matrix[i] = embedding_vector
print("Dencoder embedding shape {}".format(decoder_embedding_matrix.shape))
decoder_target_data = list()
for token_seq in tokenized_cmd_lines:
decoder_target_data.append(token_seq[1:])
padded_cmd_lines = preprocessing.sequence.pad_sequences(
decoder_target_data, maxlen=max_output_length, padding='post')
onehot_cmd_lines = utils.to_categorical(padded_cmd_lines, num_cmd_tokens)
decoder_target_data = np.array(onehot_cmd_lines)
print('Decoder target data shape -> {}'.format(decoder_target_data.shape))
encoder_inputs = tf.keras.layers.Input(shape=(None, ))
encoder_embedding = tf.keras.layers.Embedding(
num_desc_tokens, 300, weights=[encoder_embedding_matrix])(encoder_inputs)
encoder_outputs, state_h, state_c = tf.keras.layers.LSTM(
300, return_state=True)(encoder_embedding)
encoder_states = [state_h, state_c]
decoder_inputs = tf.keras.layers.Input(shape=(None, ))
decoder_embedding = tf.keras.layers.Embedding(
num_cmd_tokens, 300, weights=[decoder_embedding_matrix])(decoder_inputs)
decoder_lstm = tf.keras.layers.LSTM(300,
return_state=True,
X_train, X_test, y_train, y_test = train_test_split(data,
labels,
test_size=0.2,
random_state=68)
print(X_train.shape)
print(X_train.shape)
if (scenarioBlackAndWhite):
X_train = X_train.reshape(list(X_train.shape) + [1])
X_test = X_test.reshape(list(X_test.shape) + [1])
#Converting the labels into one hot encoding
y_train = to_categorical(y_train, classes)
y_test = to_categorical(y_test, classes)
#Building the model
model = Sequential()
model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
activation='relu',
input_shape=X_train.shape[1:]))
model.add(Conv2D(filters=32, kernel_size=(5, 5), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(rate=0.25))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
print(sequences[:5])
word_index = tokenizer.word_index
vocab_size = len(word_index) + 1
print('단어 집합(Vocabulary)의 크기 :', vocab_size)
print('문장의 최대 길이 :', max(len(l) for l in sequences))
print('문장의 평균 길이 :', sum(map(len, sequences)) / len(sequences))
plt.hist([len(s) for s in sequences], bins=50)
plt.xlabel('length of samples')
plt.ylabel('number of samples')
plt.show()
max_len = 30
intent_train = pad_sequences(sequences, maxlen=max_len)
label_train = to_categorical(np.asarray(label_train))
print('전체 데이터의 크기(shape):', intent_train.shape)
print('레이블 데이터의 크기(shape):', label_train.shape)
print(intent_train[1])
print(label_train[1])
indices = np.arange(intent_train.shape[0])
np.random.shuffle(indices)
print(indices)
intent_train = intent_train[indices]
label_train = label_train[indices]
n_of_val = int(0.1 * intent_train.shape[0])
print(n_of_val)
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # or any {'0', '1', '2'}
import tensorflow_datasets as tfds
import tensorflow as tf
import matplotlib.pyplot as plt
from plots import *
from mycallbacks import *
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import models, optimizers, regularizers, layers, callbacks
print('TF:', tf.__version__)
data_dir = os.path.join(os.path.dirname(__file__), 'data')
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
datagen_augment = ImageDataGenerator(featurewise_center= True,\
featurewise_std_normalization= True,\
width_shift_range=0.1,\
shear_range=0.1,\
zoom_range= 0.1)
datagen_non_augment = ImageDataGenerator(featurewise_center= True,\
featurewise_std_normalization= True)
datagen_augment.fit(x_train)
datagen_non_augment.fit(x_train)
train_gen = datagen_augment.flow(x_train, y_train, batch_size= 32)
val_gen = datagen_non_augment.flow(x_test, y_test, batch_size= 32)
for i in range(0, input_len - seq_length, 1):
#using sequence length, define in and out sequences
in_seq = processed_inputs[i:i + seq_length]
out_seq = processed_inputs[i + seq_length]
x_data.append([char_to_num[char] for char in in_seq])
y_data.append(char_to_num[out_seq])
n_patterns = len(x_data)
print("total patterns: ", n_patterns)
#convert into a numpy array between 0-1 which the sigmoid fn can use
X = numpy.reshape(x_data, (n_patterns, seq_length, 1))
X = X / float(vocab_len)
#hot-one encoding
y = utils.to_categorical(y_data)
#Now define our nn model
model = Sequential()
model.add(
LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(256, return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(128))
model.add(Dropout(0.2))
model.add(Dense(y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
#Now we save the weights for the future
(x_train, y_train), (x_test, y_test) = mnist.load_data()
print(x_train.shape, y_train.shape) # (60000, 28, 28), (60000,)
print(x_test.shape, y_test.shape) # (10000, 28, 28), (10000,)
print(x_train[0])
print(y_train[0])
print(x_train[0].shape) # (28, 28)
x_train = x_train.reshape(60000, 28, 28, 1) / 255.
x_test = x_test.reshape(10000, 28, 28, 1) / 255.
from tensorflow.keras.utils import to_categorical
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Flatten, Dropout
from tensorflow.keras.layers import BatchNormalization, Activation
from tensorflow.keras.regularizers import l1, l2, l1_l2
model = Sequential()
model.add(
Conv2D(filters=256,
kernel_size=(2, 2),
padding='same',
strides=1,
input_shape=(28, 28, 1)))
model.add(BatchNormalization())
from tensorflow.keras.utils import to_categorical
import os
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
data_train = pd.read_csv(
"/home/razorback/BTP/metamorphic_testing/MNIST/fashionmnist/fashion-mnist_train.csv"
)
data_test = pd.read_csv(
"/home/razorback/BTP/metamorphic_testing/MNIST/fashionmnist/fashion-mnist_test.csv"
)
img_rows, img_cols = 28, 28
input_shape = (img_rows, img_cols, 1)
X = np.array(data_train.iloc[:, 1:])
y = to_categorical(np.array(data_train.iloc[:, 0]))
#Here we split validation data to optimiza classifier during training
X_train, X_val, y_train, y_val = train_test_split(X,
y,
test_size=0.2,
random_state=13)
#Test data
X_test = np.array(data_test.iloc[:, 1:])
y_test = to_categorical(np.array(data_test.iloc[:, 0]))
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1)
X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)
X_val = X_val.reshape(X_val.shape[0], img_rows, img_cols, 1)
#https://www.youtube.com/watch?v=V23DmbdzMvg import tensorflow as tf from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from tensorflow.keras.utils import to_categorical from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn import preprocessing iris = load_iris() X = preprocessing.scale(iris['data']) Y = to_categorical(iris['target']) #print(X) #print(Y) #training data and test data X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2) #model model = Sequential() model.add(Dense(10, input_dim=4, activation='relu')) model.add(Dense(10, activation='relu')) model.add(Dense(3, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) #fitting the model model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=200, batch_size=10)
from sklearn.metrics import accuracy_score, f1_score, roc_auc_score
import matplotlib.pyplot as plt
from sklearn.model_selection import GridSearchCV
import pandas as pd
import numpy as np
data = fetch_openml('hls4ml_lhc_jets_hlf')
x_data, y_data = data['data'], data['target']
# ======================= PREPROCESSING ==============================
# encode data using one-hot, i.e turn 1 column of categorical labels into
# 5 columns of dummy varaiables
# also split data into 80-20 train test split
le = LabelEncoder()
y_data_one_hot = le.fit_transform(y_data)
y_data_one_hot = to_categorical(y_data_one_hot, 5)
x_train, x_test, y_train_one_hot, y_test_one_hot = train_test_split(x_data,
y_data_one_hot,
test_size=0.2,
random_state=42)
x_train, x_test, y_train, y_test = train_test_split(x_data, y_data,
test_size=0.2,
random_state=42)
# normalize our data (using standard normal distribution - Gaussian)
scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)
