# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
print('Evaluate IRNN...')
model = Sequential()
model.add(
SimpleRNN(hidden_units,
kernel_initializer=initializers.RandomNormal(stddev=0.001),
recurrent_initializer=initializers.Identity(gain=1.0),
activation='relu',
input_shape=x_train.shape[1:]))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
rmsprop = RMSprop(learning_rate=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
scores = model.evaluate(x_test, y_test, verbose=0)
print('IRNN test score:', scores[0])
print('IRNN test accuracy:', scores[1])
def main():
# path = get_file('nietzsche.txt', origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
path = 'data/nishinokana.txt'
wdict = WDict(path)
data = []
for line in open(path, encoding='utf-8'):
line = mecab.parse(line)
if line.strip() == '':
continue
line += '_n'
s = line.strip().split()
# s = ['<s>'] + s + ['</s>']
enc = wdict.encode(s)
data.append(enc)
data_flatten = sum(data, [])
# data = sum(data, [])
# print('corpus length:', len(text))
# chars = sorted(list(set(text)))
# print('total chars:', len(chars))
# char_indices = dict((c, i) for i, c in enumerate(chars))
# indices_char = dict((i, c) for i, c in enumerate(chars))
# cut the text in semi-redundant sequences of maxlen characters
maxlen = 30
step = 1
sentences = []
next_words = []
for i in range(0, len(data_flatten) - maxlen, step):
sentences.append(data_flatten[i: i + maxlen])
next_words.append(data_flatten[i + maxlen])
print('nb sequences:', len(sentences))
print('Vectorization...')
# X = np.zeros((len(sentences), maxlen, wdict.num_words()), dtype=np.bool)
# y = np.zeros((len(sentences), wdict.num_words()), dtype=np.bool)
# for i, sentence in enumerate(sentences):
# for t, word in enumerate(sentence):
# X[i, t, word] = 1
# y[i, next_words[i]] = 1
X = np.array(sentences, dtype=np.int)
y = np.array(next_words, dtype=np.int)
# y = np.zeros((len(sentences), wdict.num_words()), dtype=np.bool)
# for i, word in enumerate(next_words):
# y[i, word] = 1
# y = tf.one_hot(next_words, wdict.num_words())
# build the model: a single LSTM
print('Build model...')
if os.path.exists(model_filepath):
print('load model')
model = keras.models.load_model(model_filepath)
else:
model = Sequential()
model.add(Embedding(wdict.num_words(), 128, input_length=maxlen))
# model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(LSTM(64, input_shape=(maxlen, 128)))
model.add(Dropout(0.2))
model.add(Dense(wdict.num_words()))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
def sample(preds, temperature=1.0):
# helper function to sample an index from a probability array
preds = np.asarray(preds).astype('float64')
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
preds = exp_preds / np.sum(exp_preds)
probas = np.random.multinomial(1, preds, 1)
return np.argmax(probas)
# train the model, output generated text after each iteration
for iteration in range(1, 60000):
print()
print('-' * 50)
print('Iteration', iteration)
batch_size = 128
counter = 0
for i in range(0, len(X), batch_size):
y_ = to_one_hot(y[i:i+batch_size], wdict.num_words())
loss = model.train_on_batch(
X[i:i+batch_size],
y_
)
if counter % 10 == 0:
print("batch: {}/{}, loss: {}".format(i, len(X), loss))
counter += 1
# model.fit(X, y,
# batch_size=128,
# epochs=1)
model.save(model_filepath, overwrite=True)
start_index = random.randint(0, len(data_flatten) - maxlen - 1)
for diversity in [0.2, 0.5, 1.0, 1.2]:
print()
print('----- diversity:', diversity)
generated = ''
sentence = data_flatten[start_index: start_index + maxlen]
generated += wdict.decode(sentence)
print('----- Generating with seed: "' + generated + '"')
sys.stdout.write(generated)
for i in range(400):
# x = np.zeros((1, maxlen, wdict.num_words()))
x = np.array([sentence], dtype=np.int)
# for t, word in enumerate(sentence):
# x[0, t, word] = 1.
# ipdb.set_trace()
preds = model.predict(x, verbose=0)[0]
next_index = sample(preds, diversity)
next_word = wdict.to_w(next_index)
generated += next_word
sentence = sentence[1:] + [next_index]
if next_word == '_n':
print()
else:
sys.stdout.write(next_word)
sys.stdout.flush()
print()
train_file = os.path.join('data', 'train.csv')
x_train, y_train = load_train_data(train_file, normalize=False)
x_train = (x_train.astype(np.float32) - 127.5) / 127.5
x_train = np.reshape(x_train, (x_train.shape[0], 28, 28, 1))
batch_size = 32
epochs = 4000
latent_dim = 100
sample_interval = 50
channels = 1
n_critic = 5
clip_value = 0.01
optimizer = RMSprop(lr=0.00005)
valid = -np.ones((batch_size, 1))
fake = np.ones((batch_size, 1))
discriminator = create_discriminator()
discriminator.compile(loss=wasserstein_loss,
optimizer=optimizer,
metrics=['accuracy'])
discriminator.trainable = False
generator = create_generator(latent_dim, )
z = Input(shape=(latent_dim, ))
img = generator(z)
from keras.models import Sequential from keras import layers from keras.optimizers import RMSprop model = Sequential() model.add(layers.Flatten(input_shape=(lookback // step, float_data.shape[-1]))) model.add(layers.Dense(32, activation='relu')) model.add(layers.Dense(1)) model.compile(optimizer=RMSprop(), loss='mae') history = model.fit_generator(train_gen, steps_per_epoch=500, epochs=20, validation_data=val_gen, validation_steps=val_steps) # plot the loss curves for validation and training import matplotlib.pyplot as plt loss = history.history['loss'] val_loss = history.history['val_loss'] epochs = range(1, len(loss) + 1) plt.figure()
def main(input_shape):
optim = RMSprop(lr=p.LEARNING_RATE, rho = 0.9, epsilon=1e-06)
model = all_models.model_default(input_shape)
action_states = [[0,1],[1,0]]
gameState = game.GameState()
highestScore = 0
totScore = 0
if p.TRAIN_PRETRAINED and p.LOAD_POPULATED_QUEUE:
if not os.path.isfile(p.TRAIN_PRETRAINED_PATH):
print 'Pretrained Weights not found. Check the path provided.'
sys.exit(1)
model.load_weights(p.TRAIN_PRETRAINED_PATH)
model.compile(loss = clipped_max_objective, optimizer=optim)
print 'Loading expereince queue from disk..should take 1-2 mins...'
with open('saved_DDQN/double_dqn_queue.pkl','r') as f:
EXPERIENCE_MEMORY = cPickle.load(f)
epsilon = EXPERIENCE_MEMORY[-1][-1]
exp_num=int(p.TRAIN_PRETRAINED_PATH.split('_')[-1])
updateCount = exp_num/3000
p.POPULATE = 0
elif p.LOAD_POPULATED_QUEUE and not p.PRETRAINED:
model.compile(loss = clipped_max_objective, optimizer=optim)
print 'Loading expereince queue from disk..should take 1-2 mins...'
with open('saved_DDQN/double_dqn_queue.pkl','r') as f:
EXPERIENCE_MEMORY = cPickle.load(f)
epsilon = p.INITIAL_EPSILON
exp_num=0
highestScore = 0
updateCount = 0
p.POPULATE = 0
totScore=0
elif p.PRETRAINED:
model.compile(loss = clipped_max_objective, optimizer=optim)
if not os.path.isfile(p.PRETRAINED_PATH):
print 'Pretrained Weights not found. Check the path provided.'
sys.exit(1)
rgbDisplay, reward, tState = gameState.frame_step(np.array([0,1]))
grayDisplay = np.dot(imresize(rgbDisplay, (80,80), interp='bilinear')[:,:,:3], [0.299, 0.587, 0.114])
grayDisplay = [grayDisplay for _ in range(p.HISTORY)]
input_state = np.stack(grayDisplay, axis=2).reshape((1,p.HISTORY)+grayDisplay[0].shape)
run_pretrained(input_state,model,action_states,gameState)
else:
EXPERIENCE_MEMORY = deque(maxlen = p.EXPERIENCE_SIZE)
model.compile(loss = clipped_max_objective, optimizer=optim)
exp_num = 0
epsilon = p.INITIAL_EPSILON ## epsilon is probability with which we will choose network output
totScore = 0
updateCount = 0
## Sanity Check
print colored('Sanity Check...','green')
rgbDisplay, reward, tState = gameState.frame_step(np.array([0,1]))
grayDisplay = np.dot(imresize(rgbDisplay, (80,80), interp='bilinear')[:,:,:3], [0.299, 0.587, 0.114])
grayDisplay = [grayDisplay for _ in range(p.HISTORY)]
input_state = np.stack(grayDisplay, axis=2).reshape((1,p.HISTORY)+grayDisplay[0].shape)
model.predict(input_state,batch_size=1,verbose=1)
if p.PRETRAINED and os.path.isfile(p.PRETRAINED_PATH):
run_pretrained(input_state,model,action_states,gameState)
print 'Saving Model Architecture to file...model_arch.json\n'
with open('saved_DDQN/model_arch.json','w') as f:
f.write(model.to_json())
#Create target network
target_model = deepcopy(model)
######################################## Populate experience dataset #############################################################################################################
#Save time by loading a populated queue
try:
while '545 grade' != 'A+':
while p.POPULATE:
print 'Take a coffee break while the network populates the replay database. %d experiences to go...\n\n' %(p.POPULATE)
nn_out = model.predict(input_state,batch_size=1,verbose=0)
nn_action = [[0,0]]
nn_action[0][np.argmax(nn_out)] =1
assert(len(nn_action+action_states)==3)
action,rand_flag = select_action(nn_action+action_states,prob=[epsilon,(1-epsilon)/7,(1-epsilon)*6/7])
rgbDisplay, reward, tState = gameState.frame_step(action)
grayDisplay = (np.dot(np.fliplr(imrotate(imresize(rgbDisplay, (80,80), interp='bilinear'), -90))[:,:,:3], [0.299, 0.587, 0.114])).reshape((1,1,80,80))
output_state = np.append(grayDisplay,input_state[:,:p.HISTORY-1,:,:], axis=1)
EXPERIENCE_MEMORY.append(tuple((input_state,action,reward,output_state,tState,epsilon)))
print 'MODE : ' +colored('POPULATE\n','blue',attrs=['bold'])+ 'EXPERIENCE # : %d\t EPSILON: %f (fixed)\t'%(exp_num,epsilon) + 'REWARD : ' + colored('NA. Let birdy flap around for a while','magenta',attrs=['bold']) + '\t Max Q : %f'%nn_out.max()
p.POPULATE-=1
input_state = output_state
if not p.POPULATE:
with open('saved_DDQN/double_dqn_queue.pkl','wb') as f:
cPickle.dump(EXPERIENCE_MEMORY,f,protocol=cPickle.HIGHEST_PROTOCOL)
#############################################################################################################################################################################
######################################################### TRAIN #############################################################################################################
## Get new state!
print input_state.shape
nn_out = model.predict(input_state,batch_size=1,verbose=0)
nn_action = [[0,0]]
nn_action[0][np.argmax(nn_out)] =1
action,rand_flag = select_action(nn_action+action_states,prob=[epsilon,(1-epsilon)*1/7,(1-epsilon)*6/7])
rgbDisplay, reward, tState = gameState.frame_step(action)
grayDisplay = (np.dot(np.fliplr(imrotate(imresize(rgbDisplay, (80,80), interp='bilinear'), -90))[:,:,:3], [0.299, 0.587, 0.114])).reshape((1,1,80,80))
# plt.imshow(grayDisplay[0,0,:])
# plt.show()
output_state = np.append(grayDisplay,input_state[:,:p.HISTORY-1,:,:], axis=1)
if reward==1:
rewString = colored(str(reward),'green',attrs=['bold'])
totScore +=1
elif 0<reward<1:
rewString = colored(str(reward),'yellow',attrs=['bold'])
else:
totScore = 0
rewString = colored(str(reward),'red',attrs=['bold'])
if action[1]==1:
_act = '(flap)'
elif action[0]==1:
_act = '(nothing)'
else:
print 'Invalid Action'
sys.exit()
#No need to check if experience memory is full since duh, its a deque. EDIT : For memory reasons, im limiting the size
if len(EXPERIENCE_MEMORY) > 50000:
EXPERIENCE_MEMORY.popleft()
EXPERIENCE_MEMORY.append(tuple((input_state,action,reward,output_state,tState,epsilon)))
else:
EXPERIENCE_MEMORY.append(tuple((input_state,action,reward,output_state,tState,epsilon)))
print '\n\nMODE : ' +colored('TRAINING HARD *eye of the tiger theme playing in the distance*\n','blue',attrs=['bold'])+ '(%d **) EXPERIENCE # : %d\t EPSILON : %f\t ACTION: %s\t REWARD : %s\t Q-Value : %f\t\t Game Score : %s \t Highest Score : %d\t\t Updated Target %d times...' %(len(EXPERIENCE_MEMORY),exp_num, epsilon,colored('predicted '+_act,'green',attrs=['bold']) if rand_flag==0 else 'random '+_act,rewString,nn_out.max(),colored(str(totScore),'green',attrs=['dark']) if totScore >0 else colored(str(totScore),'grey'),highestScore,updateCount)
#Get mini-batch & GRAD DESCENT!
mini_batch = random.sample(EXPERIENCE_MEMORY,p.batch_size)
#Get target values for each experience in minibatch
targets,mini_batch_inputs = get_targets(mini_batch,target_model,model)
#predict on batch i.e. test_on_batch
loss = model.train_on_batch(mini_batch_inputs,targets)
print 'Loss : ' ,colored(str(loss),'cyan',attrs=['bold'])
#Tread lightly and increase thine greediness gently, for there is enough in this game for birdy's need but not for birdy's greed!
if epsilon <= p.FINAL_EPSILON:
epsilon += p.EPSILON_CHANGE
#update target networks and write score to file
if exp_num % p.TARGET_UPDATE_FREQ ==0:
target_model = deepcopy(model)
with open('score_details.log','a+') as f:
toWrite = '(%d **) EXPERIENCE # : %d\t EPSILON : %f\t ACTION: %s REWARD : %s\t Q-Value : %f\tGame Score : %s \t Highest Score : %d\t Updated Target %d times...\n\n\n' %(len(EXPERIENCE_MEMORY),exp_num, epsilon,'predicted '+_act if rand_flag==0 else 'random '+_act, rewString, nn_out.max(),str(totScore) if totScore >0 else str(totScore), highestScore,updateCount)
f.write(toWrite)
updateCount +=1
# Save network periodically
if exp_num % p.SAVE_NETWORK_FREQ ==0:
model.save_weights('saved_DDQN/DDQN_weights_iter_%d'%exp_num, overwrite= True)
if exp_num % p.SAVE_QUEUE_FREQ == 0:
save_queue(EXPERIENCE_MEMORY)
pass
#Update highest Score
if totScore > highestScore:
highestScore = totScore
input_state = output_state
exp_num+=1
except KeyboardInterrupt:
if raw_input('\nSave queue to file (Y/N) : ') != 'n':
print '\n\n\nSaving queue to file before quitting...takes 1-2 mins'
save_queue(EXPERIENCE_MEMORY)
print 'Saved queue. Quitting...'
else:
print 'Queue NOT saved. Quitting...'
except Exception as e:
model.save_weights('saved_DDQN/DDQN_weights_iter_%d'%exp_num, overwrite= True)
print e.message
def on_epoch_end(self, batch, logs={}):
timeTaken = time.time() - self.epoch_time_start
self.times.append(timeTaken)
print('Epoch time taken: ' + str(timeTaken) + 's')
#Set the respective hyperparameters
batch_size = 128
num_classes = 10
epochs = 50
num_hidden_layers = 1
#relu, tanh, sigmoid, softmax
activationFN = 'relu'
#RMSprop(), SGD, Adam, Adagrad, Adadelta
optimiserFN = RMSprop()
# categorical_crossentropy, binary_crossentropy
lossFN = 'categorical_crossentropy'
# File name for the generated model, figures, and diagrams.
short_name = 'MNIST_NN'
file_name = short_name + '_' + str(num_hidden_layers) + 'hiddenlayers_' + activationFN + '_' + str(epochs) + 'epochs'
# spilt the data between training and testing sets.
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Preprocess input data
# Flatten 28x28 images to 784 pixels.
x_train = x_train.reshape(60000, 784)
1))(x) # unflatten it to be dotted with the story later
print(
f'Reshaped story_weights.shape {story_weights.shape} so as to dot it with embedded_story'
)
x = Dot(axes=1)([story_weights, embedded_story])
print(f'Shape after dotting story_weights with embedded_story {x.shape}')
x = Reshape((embedding_dim, ))(x)
print(f'Reshaped to {x.shape} for passing through dense layer')
ans = Dense(vocab_size, activation='softmax')(x)
print(ans.shape)
model = Model([input_story, input_question_], ans)
# compile the model
model.compile(optimizer=RMSprop(lr=1e-2),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# train the model
r = model.fit([stories_train, queries_train],
answers_train,
epochs=4,
batch_size=32,
validation_data=([stories_test, queries_test], answers_test))
# Check how we weight each input sentence given a story and question
debug_model = Model([input_story, input_question_], story_weights)
# choose a random story
story_idx = np.random.choice(len(train_data))
'''data generator for fit_generator'''
n = 3000
i = 0
idxs = np.arange(n)
while True:
x, y = [], []
for b in range(batch_size):
if i % n == 0:
np.random.shuffle(idxs)
fn = f'{path}/x/{idxs[i]}.jpg'
im = cv2.imread(fn)
x.append(im)
fn = f'{path}/y/{idxs[i]}.jpg'
im = cv2.imread(fn, 0)
y.append(im)
i = (i + 1) % n
x = np.array(x, dtype='float32') / 255
y = np.array(y, dtype='float32') / 255
y.shape = y.shape + (1,)
yield x, y
model = get_unet_512()
reduce_lr = ReduceLROnPlateau(verbose=1)
model.compile(optimizer=RMSprop(), loss='mean_absolute_error', metrics=[dice_coeff, psnr])
model.fit_generator(data_generator('../dataset', 2),
steps_per_epoch=1500,
epochs=10,
callbacks=[ModelCheckpoint('./weights.{epoch:03d}.h5'), reduce_lr])
model.save('model.h5', include_optimizer=False)
y_test = keras.utils.to_categorical( y_test, num_classes)
print('y_train[0]=', y_train[0])
print('y_test [0]=', y_test[0])
print('\ncreate model...')
model = Sequential()# stack of layers
#model.add(tf.keras.layers.Flatten())
model.add(Dense (num_hidden, activation='relu', input_shape=(num_input,)))
model.add(Dropout(0.2))
model.add(Dense (num_hidden, activation='relu'))
model.add(Dropout(0.2))#regularization technic by removing some nodes
model.add(Dense (num_classes, activation='softmax'))# last layer always has softmax(except for regession problems and binary- 2 classes where sigmoid is enough)
# Prints a string summary of the neural network.')
model.summary()
model.compile(loss = 'categorical_crossentropy',# measure how accurate the model during training
optimizer = RMSprop(),#this is how model is updated based on data and loss function
metrics = ['accuracy'])
print('\ntrain model...')
history = model.fit( x_train
, y_train
, batch_size = batch_size
, epochs = epochs
, validation_data= (x_test, y_test)
, verbose = 1
# , callbacks=[early_stop, PrintDot()]#Early stopping is a useful technique to prevent overfitting.
)
print('\nplot_accuracy_loss_vs_time...')
def __init__(self, inp_dim, out_dim, lr):
self.inp_dim = inp_dim
self.out_dim = out_dim
self.rms_optimizer = RMSprop(lr=lr, epsilon=0.1, rho=0.99)
X_train = X_train_1d.astype('float32')
X_test = X_test_1d.astype('float32')
from keras.utils.np_utils import to_categorical
y_train_cat = to_categorical(y_train)
y_test_cat = to_categorical(y_test)
from keras.models import Sequential
from keras.layers import Dense
model = Sequential()
model.add(Dense(units=512, input_dim=28 * 28, activation='relu'))
model.add(Dense(units=256, activation='relu'))
model.add(Dense(units=128, activation='relu'))
model.add(Dense(units=32, activation='relu'))
model.add(Dense(units=10, activation='softmax'))
model.summary()
from keras.optimizers import RMSprop
model.compile(optimizer=RMSprop(),
loss='categorical_crossentropy',
metrics=['accuracy'])
h = model.fit(X_train, y_train_cat, epochs=10)
scores = model.evaluate(X_test, y_test_cat, verbose=0)
print(scores[1] * 100)
# In[3]:
if i == 2:
from keras.datasets import mnist
dataset = mnist.load_data('mymnist.db')
train, test = dataset
X_train, y_train = train
X_test, y_test = test
#img1 = X_train[7]
def build_cyclegan(shapes,
source_name='source',
target_name='target',
kernel_size=3,
patchgan=False,
identity=False):
"""Build the CycleGAN
1) Build target and source discriminators
2) Build target and source generators
3) Build the adversarial network
Arguments:
shapes (tuple): source and target shapes
source_name (string): string to be appended on dis/gen models
target_name (string): string to be appended on dis/gen models
kernel_size (int): kernel size for the encoder/decoder or dis/gen
models
patchgan (bool): whether to use patchgan on discriminator
identity (bool): whether to use identity loss
Returns:
(list): 2 generator, 2 discriminator, and 1 adversarial models
"""
source_shape, target_shape = shapes
lr = 2e-4
decay = 6e-8
gt_name = "gen_" + target_name
gs_name = "gen_" + source_name
dt_name = "dis_" + target_name
ds_name = "dis_" + source_name
# build target and source generators
g_target = build_generator(source_shape,
target_shape,
kernel_size=kernel_size,
name=gt_name)
g_source = build_generator(target_shape,
source_shape,
kernel_size=kernel_size,
name=gs_name)
print('---- TARGET GENERATOR ----')
g_target.summary()
print('---- SOURCE GENERATOR ----')
g_source.summary()
# build target and source discriminators
d_target = build_discriminator(target_shape,
patchgan=patchgan,
kernel_size=kernel_size,
name=dt_name)
d_source = build_discriminator(source_shape,
patchgan=patchgan,
kernel_size=kernel_size,
name=ds_name)
print('---- TARGET DISCRIMINATOR ----')
d_target.summary()
print('---- SOURCE DISCRIMINATOR ----')
d_source.summary()
optimizer = RMSprop(lr=lr, decay=decay)
d_target.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
d_source.compile(loss='mse', optimizer=optimizer, metrics=['accuracy'])
d_target.trainable = False
d_source.trainable = False
# build the computational graph for the adversarial model
# forward cycle network and target discriminator
source_input = Input(shape=source_shape)
fake_target = g_target(source_input)
preal_target = d_target(fake_target)
reco_source = g_source(fake_target)
# backward cycle network and source discriminator
target_input = Input(shape=target_shape)
fake_source = g_source(target_input)
preal_source = d_source(fake_source)
reco_target = g_target(fake_source)
# if we use identity loss, add 2 extra loss terms
# and outputs
if identity:
iden_source = g_source(source_input)
iden_target = g_target(target_input)
loss = ['mse', 'mse', 'mae', 'mae', 'mae', 'mae']
loss_weights = [1., 1., 10., 10., 0.5, 0.5]
inputs = [source_input, target_input]
outputs = [
preal_source, preal_target, reco_source, reco_target, iden_source,
iden_target
]
else:
loss = ['mse', 'mse', 'mae', 'mae']
loss_weights = [1., 1., 10., 10.]
inputs = [source_input, target_input]
outputs = [preal_source, preal_target, reco_source, reco_target]
# build adversarial model
adv = Model(inputs, outputs, name='adversarial')
optimizer = RMSprop(lr=lr * 0.5, decay=decay * 0.5)
adv.compile(loss=loss,
loss_weights=loss_weights,
optimizer=optimizer,
metrics=['accuracy'])
print('---- ADVERSARIAL NETWORK ----')
adv.summary()
return g_source, g_target, d_source, d_target, adv
def nn(train_values, train_classes_binary, test_values, test_classes_binary):
model = Sequential()
model.add(Dense(512, input_dim=820, init='uniform', activation='tanh'))
# model.add(BatchNormalization())
model.add(Dropout(0.5))
# model.add(Dense(1024, init='uniform', activation='tanh'))
# model.add(Dropout(0.5))
model.add(Dense(128, init='uniform', activation='tanh'))
# model.add(BatchNormalization())
model.add(Dropout(0.5))
# model.add(Dense(64, init='uniform', activation='tanh'))
# model.add(Dropout(0.5))
model.add(Dense(14, init='uniform', activation='softmax'))
adam = Adam()
rmsprop = RMSprop()
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error',
optimizer=sgd,
metrics=['accuracy'])
history = model.fit(
train_values,
train_classes_binary,
batch_size=16,
nb_epoch=100,
verbose=2,
# validation_split=0.1,
validation_data=(test_values, test_classes_binary),
shuffle=True)
print('training finished')
score = model.evaluate(test_values,
test_classes_binary,
batch_size=16,
verbose=1)
plt.figure(figsize=(16, 12))
# plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title('Train data vs validation data accuracy')
# plt.colorbar()
# tick_marks = numpy.arange(len(desc))
# plt.xticks(tick_marks, desc, rotation=45)
# plt.yticks(tick_marks, desc)
# plt.tight_layout()
plt.plot(list(range(1, 101)), history.history['acc'])
plt.plot(list(range(1, 101)), history.history['val_acc'])
plt.legend(['train', 'val'], loc='upper left')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
return score, history
def build_model(self, lr=1e-3):
dropout = 0.2
shape = (None, self.ydim, self.xdim, self.channels)
left = Input(batch_shape=shape)
right = Input(batch_shape=shape)
# left image as reference
x = Conv2D(filters=16, kernel_size=5, padding='same')(left)
xleft = Conv2D(filters=1,
kernel_size=5,
padding='same',
dilation_rate=2)(left)
# left and right images for disparity estimation
xin = keras.layers.concatenate([left, right])
xin = Conv2D(filters=32, kernel_size=5, padding='same')(xin)
# image reduced by 8
x8 = MaxPooling2D(8)(xin)
x8 = BatchNormalization()(x8)
x8 = Activation('relu', name='downsampled_stereo')(x8)
dilation_rate = 1
y = x8
# correspondence network
# parallel cnn at increasing dilation rate
for i in range(4):
a = Conv2D(filters=32,
kernel_size=5,
padding='same',
dilation_rate=dilation_rate)(x8)
a = Dropout(dropout)(a)
y = keras.layers.concatenate([a, y])
dilation_rate += 1
dilation_rate = 1
x = MaxPooling2D(8)(x)
# disparity network
# dense interconnection inspired by DenseNet
for i in range(4):
x = keras.layers.concatenate([x, y])
y = BatchNormalization()(x)
y = Activation('relu')(y)
y = Conv2D(filters=64, kernel_size=1, padding='same')(y)
y = BatchNormalization()(y)
y = Activation('relu')(y)
y = Conv2D(filters=16,
kernel_size=5,
padding='same',
dilation_rate=dilation_rate)(y)
y = Dropout(dropout)(y)
dilation_rate += 1
# disparity estimate scaled back to original image size
x = keras.layers.concatenate([x, y], name='upsampled_disparity')
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Conv2D(filters=32, kernel_size=1, padding='same')(x)
x = UpSampling2D(8)(x)
if not self.settings.nopadding:
x = ZeroPadding2D(padding=(2, 0))(x)
# left image skip connection to disparity estimate
x = keras.layers.concatenate([x, xleft])
y = BatchNormalization()(x)
y = Activation('relu')(y)
y = Conv2D(filters=16, kernel_size=5, padding='same')(y)
x = keras.layers.concatenate([x, y])
y = BatchNormalization()(x)
y = Activation('relu')(y)
y = Conv2DTranspose(filters=1, kernel_size=9, padding='same')(y)
# prediction
if self.settings.otanh:
yout = Activation('tanh', name='disparity_output')(y)
else:
yout = Activation('sigmoid', name='disparity_output')(y)
# densemapnet model
self.model = Model([left, right], yout)
if self.settings.model_weights:
print("Loading checkpoint model weights %s...." %
self.settings.model_weights)
self.model.load_weights(self.settings.model_weights)
if self.settings.otanh:
self.model.compile(loss='binary_crossentropy',
optimizer=RMSprop(lr=lr))
else:
self.model.compile(loss='mse', optimizer=RMSprop(lr=lr))
print("DenseMapNet Model:")
self.model.summary()
plot_model(self.model, to_file='densemapnet.png', show_shapes=True)
return self.model
def build_models(self):
lr = 4e-4
clip = 1.0
decay = 1e-8
# Initialize architectures
self.disc = self.init_discriminator()
self.gen = self.init_generator()
if self.weights_path is not None:
self.gen.load_weights(self.weights_path + 'generator_weights.h5')
self.disc.load_weights(self.weights_path +
'discriminator_weights.h5')
# Create the model
if self.num_gpus > 1:
self.disc_model = multi_gpu_model(self.disc, gpus=self.num_gpus)
else:
self.disc_model = self.disc
# Compile Discriminator Model
# doptimizer = RMSprop(lr=lr, decay=decay, clipvalue=clip)
self.disc_model.compile(loss='mse',
optimizer=SGD(lr=lr,
nesterov=True,
clipvalue=clip),
metrics=['accuracy'])
#self.disc_model = Sequential()
#self.disc_model.add(self.disc)
#self.disc_model.compile(loss='binary_crossentropy',
# optimizer = doptimizer,
# metrics=['accuracy'])
# Compile Adversarial Model
goptimizer = RMSprop(lr=lr / 2, decay=decay, clipvalue=clip)
self.disc.trainable = False
im_lr = Input(shape=(self.imlr_w, self.imlr_h, self.im_c))
im_hr = Input(shape=(self.imhr_w, self.imhr_h, self.im_c))
# Generated HR Images
gen_hr = self.gen(im_lr)
# Discriminator on Generator Output
disc_gen_hr = self.disc(gen_hr)
self.adv_model = Model(im_lr, disc_gen_hr)
# Create the model
if self.num_gpus > 1:
self.adv_model = multi_gpu_model(Model(im_lr, disc_gen_hr),
gpus=self.num_gpus)
else:
self.adv_model = Model(im_lr, disc_gen_hr)
self.adv_model.compile(
loss=['binary_crossentropy'],
#loss_weights=[1e-3,1],
optimizer=Adam(clipvalue=clip),
metrics=['accuracy'])
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=x_train.shape[1:]))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=20, verbose=2,
callbacks=KerasUtils().buildTensorflowCallback(),
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test)
model.save('../models/mnist20.h5')
print('Test loss:', score[0])
print('Test accuracy:', score[1])
def lstmModel(vectorised_data, target):
split_point = int(len(vectorised_data) * .8)
print('Split Point ', split_point)
#split data into training and testing
x_train = vectorised_data[:split_point]
y_train = target[:split_point]
x_test = vectorised_data[split_point:]
y_test = target[split_point:]
#make each point of data of uniform lenght
x_train = pad_trunc(x_train, maxlen)
x_test = pad_trunc(x_test, maxlen)
#reshape data into a numpy structure
print("X_TRAIN Reshape Started ")
print(f' Training data Size: {len(x_train)}')
print("Number of word tokens ", maxlen)
print("Embedding Dims ", embedding_dims)
#print(f'Training Data {x_train[:1]}')
#print(type(x_train))
#y_train = np.array(y_train)
x_train = np.reshape(x_train, (len(x_train), maxlen, embedding_dims))
print("X_TRAIN Reshape Completed ")
#y_train = np.array(y_train)
y_train = to_categorical(y_train, 2)
x_test = np.reshape(x_test, (len(x_test), maxlen, embedding_dims))
#y_test = np.array(y_test)
y_test = to_categorical(y_test, 2)
#print(f'Y_TEST DATA: {y_test}')
print((f'Y_TEST_DATA LENGHT{len(y_test)}'))
print("Data Reshape Ended ")
model = Sequential()
#model.add(Embedding(embedding_dims, batch_size)) print(f' Training data Size: {len(x_train)}')
#model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))
#model.add(LSTM(num_neurons, return_sequences=True, input_shape=(maxlen , embedding_dims)))
#model.add(Dense(2, activation='relu'))
#model.add(Dense(2, activation='sigmoid')) # one class
model.add(
LSTM(num_neurons,
return_sequences=True,
input_shape=(maxlen, embedding_dims),
use_bias=True)) #stack LSTMs
model.add(Dropout(.2))
model.add(Flatten()) # dense layer expects a flat vectors of n elements
#model.add(Dense(1, activation='relu')) # one class
#model.add(Dense(2, activation='relu')) # one class model.add(Dense(1, activation='relu')) # one class
#model.add(Dropout(0.2))
model.add(Dense(2, activation='sigmoid')) # two class
#model.add(Dense(2, activation='tanh')) # two class
#model.add((Dense(2)))
#model.add(Activation('softmax')) # one class
optimizer = RMSprop(
learning_rate=0.001
) # use learning rate to improve the accuracy of the model
#model.compile('rmsprop', 'binary_crossentropy', metrics=['accuracy'])
model.compile(loss='binary_crossentropy', optimizer=optimizer)
#model.compile(loss='categorical_crossentropy', optimizer=optimizer)
#model.compile('rmsprop', 'binary_crossentropy', metrics=['accuracy'])
#model.compile(sample_weight_mode="temporal"
#optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
#print(model.summary())
fitmodel(model, x_train, y_train, x_test, y_test, batch_size, epochs)
def get_unet_coords(input_shape=(256, 256, 3),
num_classes=1):
inputs = Input(shape=input_shape)
# 256
down0 = Conv2D(32, (3, 3), padding='same')(inputs)
down0 = BatchNormalization()(down0)
down0 = Activation('relu')(down0)
down0 = Conv2D(32, (3, 3), padding='same')(down0)
down0 = BatchNormalization()(down0)
down0 = Activation('relu')(down0)
down0_pool = MaxPooling2D((2, 2), strides=(2, 2))(down0)
# 128
down1 = Conv2D(64, (3, 3), padding='same')(down0_pool)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1 = Conv2D(64, (3, 3), padding='same')(down1)
down1 = BatchNormalization()(down1)
down1 = Activation('relu')(down1)
down1_pool = MaxPooling2D((2, 2), strides=(2, 2))(down1)
# 64
down2 = Conv2D(128, (3, 3), padding='same')(down1_pool)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2 = Conv2D(128, (3, 3), padding='same')(down2)
down2 = BatchNormalization()(down2)
down2 = Activation('relu')(down2)
down2_pool = MaxPooling2D((2, 2), strides=(2, 2))(down2)
# 32
down3 = Conv2D(256, (3, 3), padding='same')(down2_pool)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3 = Conv2D(256, (3, 3), padding='same')(down3)
down3 = BatchNormalization()(down3)
down3 = Activation('relu')(down3)
down3_pool = MaxPooling2D((2, 2), strides=(2, 2))(down3)
# 16
down4 = Conv2D(512, (3, 3), padding='same')(down3_pool)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4 = Conv2D(512, (3, 3), padding='same')(down4)
down4 = BatchNormalization()(down4)
down4 = Activation('relu')(down4)
down4_pool = MaxPooling2D((2, 2), strides=(2, 2))(down4)
# 8
center = Conv2D(8, (3, 3), padding='same')(down4_pool)
center = BatchNormalization()(center)
center = Activation('relu')(center)
center = Conv2D(8, (3, 3), padding='same')(center)
center = BatchNormalization()(center)
center = Activation('relu')(center)
classify = Flatten()(center)
#classify = Dense(32)(center)
classify = Dense(8, activation="sigmoid", use_bias=False)(classify)
model = Model(inputs=inputs, outputs=classify)
model.compile(optimizer=RMSprop(lr=0.0001), loss="mean_squared_error", metrics=["mse"])
return model
def generate_model(feat_dict):
con_feats = feat_dict['con_feats']
lstm_feats = feat_dict['lstm_feats']
M = feat_dict['M']
# Initialize input
INPUT = [[], [], []]
# A) USER PROFILE FEATURE LAYERS
# categorical features
for i, m in enumerate(M):
INPUT[0].append(Input(shape=(None, 1),
name='cat_' + str(i) + '_input'))
INPUT[1].append(
Embedding(m[0], min(m),
name='cat_' + str(i) + '_embedding')(INPUT[0][-1]))
INPUT[2].append(
Reshape((-1, min(m)),
name='cat_' + str(i) + '_reshape')(INPUT[1][-1]))
# continuous features
cont_input = Input(shape=(None, len(con_feats)), name='cont_input')
INPUT[2].append(cont_input)
# input concatenation
concat1 = Concatenate(name='profile_concat')(INPUT[2])
# B) LSTM LAYERS
lstm_input = Input(shape=(None, len(lstm_feats)), name='lstm_input')
# lstm1 from input lstm_input
lstm1 = LSTM(80,
recurrent_regularizer=L1L2(),
dropout=.1,
return_sequences=True,
input_shape=(None, len(lstm_feats)),
name='lstm_layer_1')(lstm_input)
# lstm2 from input lstm1
lstm2 = LSTM(32,
recurrent_regularizer=L1L2(),
dropout=.1,
return_sequences=True,
name='lstm_layer_2')(lstm1)
concat2 = Concatenate(name='profile_lstm_concat')(INPUT[2] + [lstm2])
# C) DENSE LAYERS
# dns1 from concat2
dns1 = Dense(128 * 3, name='dense_layer_1')(concat2)
dns1 = LeakyReLU()(dns1)
concat3 = Dropout(.1)(dns1)
# dns2 from dns1
dns2 = Dense(128 * 1, name='dense_layer_2')(concat3)
dns2 = LeakyReLU()(dns2)
# drpt1 from dns2
drpt1 = Dropout(.2)(dns2)
# dns3 from drpt1
dns3 = Dense(1, name='dense_layer_3')(drpt1)
# MODEL
model = Model(INPUT[0] + [cont_input, lstm_input], dns3)
print(model.summary(line_length=80))
rms_prop = RMSprop(lr=0.0001)
model.compile(
rms_prop,
loss='mean_squared_error',
metrics=['mean_squared_error', 'mean_absolute_error'],
sample_weight_mode='temporal') # timestep-wise sample weighting
return model
outputDepth = 6
model = Sequential()
model.add(
Dense(164,
kernel_initializer='lecun_uniform',
input_shape=((dataCount * dataDepth), )))
model.add(Activation('relu'))
# Hidden layer
model.add(Dense(150, kernel_initializer='lecun_uniform'))
model.add(Activation('relu'))
# Output layer, use linear so they're real world values
model.add(Dense(6, kernel_initializer='lecun_uniform'))
model.add(Activation('linear'))
rms = RMSprop()
# Next try
model.compile(loss='binary_crossentropy', optimizer=rms)
# Functions
#Calculated the time difference in milliseconds
#Regex on each filepath to get the time variables
#Create an object to get the epoch time
#Differene in epoch time is the dt (in seconds)
def calculateDt(prev, current):
#Calculate the epoch of the prev tick
#File last value is millisconds so convert to microseconds in datetime constructor
p = re.search(r'.+center_(\d+)_(\d+)_(\d+)_(\d+)_(\d+)_(\d+)_(\d+).jpg',
prev)
input = open('../Pre/X.pkl', 'rb')
X_mean = sPickle.load(input)
X_std = sPickle.load(input)
X_train_N = sPickle.load(input)
X_train_V = sPickle.load(input)
X_test_N = sPickle.load(input)
X_test_V = sPickle.load(input)
input.close()
input = open('../Pre/Y.pkl', 'rb')
Y_mean = sPickle.load(input)
Y_std = sPickle.load(input)
Y_train = sPickle.load(input)
Y_test = sPickle.load(input)
input.close()
model = model_from_json(open("mlp_architecture.json").read())
model.load_weights('mlp_weights.h5')
model.compile(loss='mse', optimizer=RMSprop())
X_test_V = number2vector(featnamearray, feattypedict, featdict, X_test_V)
X_test = np.hstack((X_test_V, X_test_N))
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score)
Y_predict = model.predict(X_test, verbose=0)
Y_predict = (Y_predict * Y_std) + Y_mean
Y_predict = Y_predict.astype(np.float32)
Y_predict.tofile("Y_predict")
def train_on_texts(self,
texts,
context_labels=None,
batch_size=128,
num_epochs=50,
verbose=1,
new_model=False,
gen_epochs=1,
train_size=1.0,
max_gen_length=300,
validation=True,
dropout=0.0,
via_new_model=False,
save_epochs=0,
multi_gpu=False,
**kwargs):
if new_model and not via_new_model:
self.train_new_model(texts,
context_labels=context_labels,
num_epochs=num_epochs,
gen_epochs=gen_epochs,
batch_size=batch_size,
dropout=dropout,
validation=validation,
save_epochs=save_epochs,
multi_gpu=multi_gpu,
**kwargs)
return
if context_labels:
context_labels = LabelBinarizer().fit_transform(context_labels)
if 'prop_keep' in kwargs:
train_size = prop_keep
if self.config['word_level']:
texts = [text_to_word_sequence(text, filters='') for text in texts]
# calculate all combinations of text indices + token indices
indices_list = [
np.meshgrid(np.array(i), np.arange(len(text) + 1))
for i, text in enumerate(texts)
]
indices_list = np.block(indices_list)
# If a single text, there will be 2 extra indices, so remove them
# Also remove first sequences which use padding
if self.config['single_text']:
indices_list = indices_list[self.config['max_length']:-2, :]
indices_mask = np.random.rand(indices_list.shape[0]) < train_size
if multi_gpu:
num_gpus = len(K.tensorflow_backend._get_available_gpus())
batch_size = batch_size * num_gpus
gen_val = None
val_steps = None
if train_size < 1.0 and validation:
indices_list_val = indices_list[~indices_mask, :]
gen_val = generate_sequences_from_texts(texts, indices_list_val,
self, context_labels,
batch_size)
val_steps = max(
int(np.floor(indices_list_val.shape[0] / batch_size)), 1)
indices_list = indices_list[indices_mask, :]
num_tokens = indices_list.shape[0]
assert num_tokens >= batch_size, "Fewer tokens than batch_size."
level = 'word' if self.config['word_level'] else 'character'
print("Training on {:,} {} sequences.".format(num_tokens, level))
steps_per_epoch = max(int(np.floor(num_tokens / batch_size)), 1)
gen = generate_sequences_from_texts(texts, indices_list, self,
context_labels, batch_size)
base_lr = 4e-3
# scheduler function must be defined inline.
def lr_linear_decay(epoch):
return (base_lr * (1 - (epoch / num_epochs)))
if context_labels is not None:
if new_model:
weights_path = None
else:
weights_path = "{}_weights.hdf5".format(self.config['name'])
self.save(weights_path)
self.model = textgenrnn_model(self.num_classes,
dropout=dropout,
cfg=self.config,
context_size=context_labels.shape[1],
weights_path=weights_path)
model_t = self.model
if multi_gpu:
# Do not locate model/merge on CPU since sample sizes are small.
parallel_model = multi_gpu_model(self.model,
gpus=num_gpus,
cpu_merge=False)
parallel_model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(lr=4e-3, rho=0.99))
model_t = parallel_model
print("Training on {} GPUs.".format(num_gpus))
model_t.fit_generator(gen,
steps_per_epoch=steps_per_epoch,
epochs=num_epochs,
callbacks=[
LearningRateScheduler(lr_linear_decay),
generate_after_epoch(self, gen_epochs,
max_gen_length),
save_model_weights(self, num_epochs,
save_epochs)
],
verbose=verbose,
max_queue_size=10,
validation_data=gen_val,
validation_steps=val_steps)
# Keep the text-only version of the model if using context labels
if context_labels is not None:
self.model = Model(inputs=self.model.input[0],
outputs=self.model.output[1])
from cnn_functions import rate_scheduler, train_model_sample
from model_zoo import feature_net_multires_61x61 as the_model
import os
import datetime
import numpy as np
batch_size = 256
n_classes = 3
n_epoch = 25
model = the_model(n_channels=2, n_features=3, reg=1e-5)
dataset = "HeLa_all_61x61"
direc_save = "/home/nquach/DeepCell2/trained_networks/"
direc_data = "/home/nquach/DeepCell2/training_data_npz/"
optimizer = RMSprop(lr=0.001, rho=0.95, epsilon=1e-8)
lr_sched = rate_scheduler(lr=0.001, decay=0.95)
expt = "fn_multires_61x61"
iterate = 2
train_model_sample(model=model,
dataset=dataset,
optimizer=optimizer,
expt=expt,
it=iterate,
batch_size=batch_size,
n_epoch=n_epoch,
direc_save=direc_save,
direc_data=direc_data,
lr_sched=lr_sched,
rotate=True,
checkpoint = ModelCheckpoint("/home/aay/Documents/MLOps/task4/facemodel.h5",
monitor="val_loss",
mode="min",
save_best_only=True,
verbose=1)
earlystop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=3,
verbose=1,
restore_best_weights=True)
# putting call backs into a callback list
callbacks = [earlystop, checkpoint]
# using a very small learning rate
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(lr=0.001),
metrics=['accuracy'])
# Enter the total number of training and validation samples here
nb_train_samples = 200
nb_validation_samples = 40
# only training 3 EPOCHS
epochs = 3
batch_size = 16
history = model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
callbacks=callbacks,
validation_data=validation_generator,
validation_steps=nb_validation_samples //
batch_size)
for i, sentence in enumerate(sentences):
for t, char in enumerate(sentence):
if t < maxlen - 1:
X[i, t, char_indices[char]] = 1
else:
X[i, t, char_indices[char]] = 1
y[i, t + 1, char_indices[next_chars[i]]] = 1
# build the model: a single LSTM
print('Build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(maxlen, len(chars)))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
def sample(preds, temperature=1.0):
# helper function to sample an index from a probability array
preds = np.asarray(preds).astype('float64')
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
preds = exp_preds / np.sum(exp_preds)
probas = np.random.multinomial(1, preds, 1)
return np.argmax(probas)
# train the model, output generated text after each iteration
for iteration in range(1, 60):
import pandas as pandas
import os
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
np.random.seed(1617)
# training
NB_EPOCH = 20
BATCH_SIZE = 128
VERBOSE = 1
NB_CLASSES = 10 # outputs
OPTIMIZER = SGD()
N_HIDDEN = 128
VALIDATION_SPLIT = 0.2 # how much train is reserved for validation
OPTIMIZER = RMSprop() # optimizer
OPTIMIZER = Adam()
# data suffled between training and testing
(X_train, y_train), (X_test, y_test) = mnist.load_data()
# X_train is 60000 rows of 28x28 vales, reshaped into 60000xRESHAPED
RESHAPED = 784 # 28x28
X_train = X_train.reshape(60000, RESHAPED)
X_test = X_test.reshape(10000, RESHAPED)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
# normalize X_test between 0 and 1
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
Y_test = np.reshape(Y_test, (int(Y_test.shape[0]/2), 2))
model = Sequential()
model.add(LSTM(units=30, return_sequences=True, input_shape = (WINDOW, EMB_SIZE)))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(64))
model.add(BatchNormalization())
model.add(LeakyReLU())
model.add(Dense(2))
model.add(Activation('softmax'))
opt = RMSprop(lr=0.002)
reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.9, patience=30, min_lr=0.000001, verbose=1)
checkpointer = ModelCheckpoint(filepath="lolkek.hdf5", verbose=1, save_best_only=True)
model.compile(optimizer=opt,
loss='categorical_hinge',
metrics=['accuracy'])
history = model.fit(X_train, Y_train,
nb_epoch = 50,
batch_size = 16,
verbose=1,
validation_data=(X_test, Y_test),
callbacks=[reduce_lr, checkpointer],
shuffle=False)
def __init__(self, warm_start=None, epochs=10, batch_size=86):
super().__init__(warm_start)
self.epochs = epochs
self.batch_size = batch_size
if self.model is None:
self.model = Sequential()
self.model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='Same',
activation='relu',
input_shape=(28, 28, 1)))
self.model.add(
Conv2D(filters=32,
kernel_size=(5, 5),
padding='Same',
activation='relu'))
self.model.add(MaxPool2D(pool_size=(2, 2)))
self.model.add(Dropout(0.25))
self.model.add(
Conv2D(filters=64,
kernel_size=(3, 3),
padding='Same',
activation='relu'))
self.model.add(
Conv2D(filters=64,
kernel_size=(3, 3),
padding='Same',
activation='relu'))
self.model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
self.model.add(Dropout(0.25))
self.model.add(Flatten())
self.model.add(Dense(256, activation='relu'))
self.model.add(Dropout(0.5))
self.model.add(Dense(10, activation='softmax'))
# Optimizer
optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)
# Compile model
self.model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
# Data augmentation to prevent overfitting
self.datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=
False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=
10, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.1, # Randomly zoom image
width_shift_range=
0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=
0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False # randomly flip images
)
def __init__(self, batch_size):
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
# Following parameter and optimizer set as recommended in paper
self.n_critic = 5
self.clip_value = 0.01
optimizer = RMSprop(lr=0.00005)
# Build the generator and discriminator
self.generator = self.build_generator()
self.discriminator = self.build_discriminator()
#-------------------------------
# Construct Computational Graph
# for Discriminator
#-------------------------------
# Freeze generator's layers while training generator
self.generator.trainable = False
# Image input (real sample)
real_img = Input(shape=self.img_shape)
# Noise input
z_disc = Input(shape=(100, ))
# Generate image based of noise (fake sample)
fake_img = self.generator(z_disc)
# Discriminator determines validity of the real and fake images
fake = self.discriminator(fake_img)
real = self.discriminator(real_img)
# Construct weighted average between real and fake images
merged_img = RandomWeightedAverage()([real_img, fake_img])
# Determine validity of weighted sample
valid_merged = self.discriminator(merged_img)
# Use Python partial to provide loss function with additional
# 'averaged_samples' argument
partial_gp_loss = partial(self.gradient_penalty_loss,
averaged_samples=merged_img)
partial_gp_loss.__name__ = 'gradient_penalty' # Keras requires function names
self.discriminator_model = Model(inputs=[real_img, z_disc],
outputs=[real, fake, valid_merged])
self.discriminator_model.compile(loss=[
self.wasserstein_loss, self.wasserstein_loss, partial_gp_loss
],
optimizer=optimizer,
loss_weights=[1, 1, 10])
#-------------------------------
# Construct Computational Graph
# for Generator
#-------------------------------
# For the generator we freeze the discriminator's layers
self.discriminator.trainable = False
self.generator.trainable = True
# Sampled noise for input to generator
z_gen = Input(shape=(100, ))
# Generate images based of noise
img = self.generator(z_gen)
# Discriminator determines validity
valid = self.discriminator(img)
# Defines generator model
self.generator_model = Model(z_gen, valid)
self.generator_model.compile(loss=self.wasserstein_loss,
optimizer=optimizer)
print('vector...')
x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
for t, char in enumerate(sentence):
x[i, t, char_indices[char]] = 1
y[i, char_indices[next_chars[i]]] = 1
# build LSTM model
print('build model...')
model = Sequential()
model.add(LSTM(128, input_shape=(maxlen, len(chars))))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))
model.compile(optimizer=RMSprop(lr=0.01), loss='categorical_crossentropy')
model.summary()
print("*" * 40)
print(model.summary())
def sample(preds, temperature=1.0):
preds = np.asarray(preds).astype('float64')
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
preds = exp_preds / np.sum(exp_preds)
probs = np.random.multinomial(1, preds, 1)
return np.argmax(probs)
