model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(64, kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(64, kernel_initializer='he_normal'))
model.add(Activation('elu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(num_classes, kernel_initializer='he_normal'))
model.add(Activation('softmax'))
print(model.summary())
checkpoint = ModelCheckpoint('./Trained Model/Emotion_detector.h5',
#model = Sequential()
#model.add(LSTM(7, 120, return_sequences=True))
#model.add(LSTM(120, 62, return_sequences=True))
#model.add(Dropout(0.2))
#model.add(LSTM(62, 120, return_sequences=False))
#model.add(Dropout(0.2))
#model.add(Dense(120, 7))
#model.add(Activation("linear"))
#model.compile(loss="mean_squared_error", optimizer="rmsprop")
model = Sequential()
model.add(LSTM(7, 200, return_sequences=True))
model.add(LSTM(200, 200, return_sequences=False))
model.add(Dropout(0.2))
model.add(Dense(200, 7))
model.add(Activation("linear"))
model.compile(loss="mean_squared_error", optimizer="rmsprop")
def _load_data(data, n_prev=50):
"""
data should be pd.DataFrame()
"""
docX, docY = [], []
for i in range(len(data) - n_prev):
docX.append(data.iloc[i:i + n_prev].as_matrix())
docY.append(data.iloc[i + n_prev].as_matrix())
alsX = np.array(docX)
alsY = np.array(docY)
BDT_test_error = 1 - clf_tree.score(X_test, Y_test)
BDT_train_error = 1 - clf_tree.score(X_train, Y_train)
BDT_cv_error = 1 - clf_tree.score(X_CV, Y_CV)
print("The Boosted Decision tree had a Training Error of: {0}".format(
BDT_train_error))
print("The Boosted Decision tree had a Cross-Validation Error of: {0}".format(
BDT_cv_error))
print("The Boosted Decision tree had a Testing Error of: {0}".format(
BDT_test_error))
print('\n')
model = Sequential()
model.add(
Dense(input_dim=learning_in.shape[1],
output_dim=learning_in.shape[1] / 2,
init='uniform'))
model.add(Activation('sigmoid'))
model.add(
Dense(input_dim=learning_in.shape[1] / 2,
output_dim=learning_in.shape[1] / 2,
init='uniform'))
model.add(Activation('sigmoid'))
model.add(
Dense(input_dim=learning_in.shape[1] / 2, output_dim=1, init='uniform'))
model.add(Activation('sigmoid'))
model.compile(loss='mean_absolute_error',
optimizer='adam',
class_mode='binary',
metrics=['accuracy'])
model.fit(X_train,
def __create_dense_net(nb_classes, img_input, include_top, depth=40, nb_dense_block=3, growth_rate=12, nb_filter=-1,
nb_layers_per_block=-1, bottleneck=False, reduction=0.0, dropout_rate=None, weight_decay=1e-4,
subsample_initial_block=False, activation='softmax'):
''' Build the DenseNet model
Args:
nb_classes: number of classes
img_input: tuple of shape (channels, rows, columns) or (rows, columns, channels)
include_top: flag to include the final Dense layer
depth: number or layers
nb_dense_block: number of dense blocks to add to end (generally = 3)
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters. Default -1 indicates initial number of filters is 2 * growth_rate
nb_layers_per_block: number of layers in each dense block.
Can be a -1, positive integer or a list.
If -1, calculates nb_layer_per_block from the depth of the network.
If positive integer, a set number of layers per dense block.
If list, nb_layer is used as provided. Note that list size must
be (nb_dense_block + 1)
bottleneck: add bottleneck blocks
reduction: reduction factor of transition blocks. Note : reduction value is inverted to compute compression
dropout_rate: dropout rate
weight_decay: weight decay rate
subsample_initial_block: Set to True to subsample the initial convolution and
add a MaxPool2D before the dense blocks are added.
subsample_initial:
activation: Type of activation at the top layer. Can be one of 'softmax' or 'sigmoid'.
Note that if sigmoid is used, classes must be 1.
Returns: keras tensor with nb_layers of conv_block appended
'''
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
if reduction != 0.0:
assert reduction <= 1.0 and reduction > 0.0, 'reduction value must lie between 0.0 and 1.0'
# layers in each dense block
if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:
nb_layers = list(nb_layers_per_block) # Convert tuple to list
assert len(nb_layers) == (nb_dense_block), 'If list, nb_layer is used as provided. ' \
'Note that list size must be (nb_dense_block)'
final_nb_layer = nb_layers[-1]
nb_layers = nb_layers[:-1]
else:
if nb_layers_per_block == -1:
assert (depth - 4) % 3 == 0, 'Depth must be 3 N + 4 if nb_layers_per_block == -1'
count = int((depth - 4) / 3)
if bottleneck:
count = count // 2
nb_layers = [count for _ in range(nb_dense_block)]
final_nb_layer = count
else:
final_nb_layer = nb_layers_per_block
nb_layers = [nb_layers_per_block] * nb_dense_block
# compute initial nb_filter if -1, else accept users initial nb_filter
if nb_filter <= 0:
nb_filter = 2 * growth_rate
# compute compression factor
compression = 1.0 - reduction
# Initial convolution
if subsample_initial_block:
initial_kernel = (7, 7)
initial_strides = (2, 2)
else:
initial_kernel = (3, 3)
initial_strides = (1, 1)
x = Conv2D(nb_filter, initial_kernel, kernel_initializer='he_normal', padding='same',
strides=initial_strides, use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)
if subsample_initial_block:
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, bottleneck=bottleneck,
dropout_rate=dropout_rate, weight_decay=weight_decay)
# add transition_block
x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
# The last dense_block does not have a transition_block
x, nb_filter = __dense_block(x, final_nb_layer, nb_filter, growth_rate, bottleneck=bottleneck,
dropout_rate=dropout_rate, weight_decay=weight_decay)
x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D()(x)
if include_top:
x = Dense(nb_classes, activation=activation)(x)
return x
Conv2D(32, (3, 3),
padding='same',
input_shape=(IMG_ROWS, IMG_COLS, IMG_CHANNELS)))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(NB_CLASSES))
model.add(Activation('softmax'))
model.summary()
# Определив нейросеть, мы можем перейти к обучению модели. В данном случае мы выделяем контрольный набор, помимо обучающего и тестового.
# Обучающий набор нужен для обучения модели, контрольный – для выбора наилучшего подхода к обучению, а тестовый – для проверки обученной модели на новых данных.
# обучение
model.compile(loss='categorical_crossentropy',
optimizer=OPTIM,
metrics=['accuracy'])
# model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NB_EPOCH, validation_split=VALIDATION_SPLIT, verbose=VERBOSE)
# score = model.evaluate(X_test, Y_test,batch_size=BATCH_SIZE, verbose=VERBOSE)
# Creating CNN model
input_shape = (28,28,1)
number_of_classes = 10
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(number_of_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),metrics=['accuracy'])
model.summary()
history = model.fit(X_train, y_train,epochs=5, shuffle=True,
batch_size = 200,validation_data= (X_test, y_test))
model.save('trained_model.h5')
Y = e.transform(Y_obj)
# In[70]:
n_fold = 5
skf = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=seed)
# In[71]:
accuracy = []
# In[76]:
for train, val in skf.split(X, Y):
model = Sequential()
model.add(Dense(10, input_dim=12, activation="relu"))
model.add(Dense(1, activation="softmax"))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(X[train], Y[train], epochs=20, batch_size=5)
k_accuracy = "%.4f" % (model.evaluate(X[val], Y[val])[1])
accuracy.append(k_accuracy)
# In[77]:
print("/n %.f fold accuracy : " % n_fold, accuracy)
# ## 11. Basic Modeling
import pandas as pd
import numpy as np
from keras.models import Sequential
from keras.layers.core import Dense
TARGET_VARIABLE ="user_action"
TRAIN_TEST_SPLIT=0.5
HIDDEN_LAYER_SIZE=30
raw_data = pd.read_csv("data.csv")
mask = np.random.rand(len(raw_data)) < TRAIN_TEST_SPLIT
tr_dataset = raw_data[mask]
te_dataset = raw_data[∼mask]
tr_data = np.array(raw_data.drop(TARGET_VARIABLE,axis=1))
tr_labels = np.array(raw_data[[TARGET_VARIABLE]])
te_data = np.array(te_dataset.drop(TARGET_VARIABLE,axis=1))
te_labels = np.array(te_dataset[[TARGET_VARIABLE]])
ffnn = Sequential()
ffnn.add(Dense(HIDDEN_LAYER_SIZE, input_shape=(3,),activation="sigmoid"))
ffnn.add(Dense(1, activation="sigmoid"))
ffnn.compile(loss="mean_squared_error", optimizer="sgd", metrics=[’accuracy’])
ffnn.fit(tr_data, tr_labels, epochs=150, batch_size=2,verbose=1)
metrics = ffnn.evaluate(te_data, te_labels, verbose=1)
print("%s: %.2f%%" % (ffnn.metrics_names[1],metrics[1]*100))
trainingData, depth)
validationFeatureVectors, validationTargetVectors = formFeatureTargetVectors(
validationData, depth)
testingFeatureVectors, testingTargetVectors = formFeatureTargetVectors(
testingData, depth)
# Network parameters
in_out_neurons = 1
hidden_neurons = 200
batch_size = 1
# Stack the layers
model = Sequential()
model.add(
LSTM(hidden_neurons, input_dim=in_out_neurons, return_sequences=False))
model.add(Dense(in_out_neurons))
model.add(Activation("linear"))
model.compile(loss="mean_squared_error", optimizer="rmsprop")
model.fit(trainingFeatureVectors,
trainingTargetVectors,
nb_epoch=30,
validation_split=0.05)
testingPredicted = predictFuture(model, availableData, depth, nTesting)
testingActual = testingData
# Plot the results
#Plotting of the prediction output and error
outputFolderName = "Outputs/Outputs" + datetime.now().strftime(
"%Y_%m_%d_%H_%M_%S")
os.mkdir(outputFolderName)
target_size=(224, 224),
color_mode="rgb",
shuffle=False,
batch_size=32)
# load the VGG16 network, ensuring the head FC layer sets are left
# off
baseModel = VGG16(weights="imagenet",
include_top=False,
input_tensor=Input(shape=(224, 224, 3)))
# construct the head of the model that will be placed on top of the
# the base model
headModel = baseModel.output
headModel = Flatten(name="flatten")(headModel)
headModel = Dense(512, activation="relu")(headModel)
headModel = Dropout(0.5)(headModel)
headModel = Dense(len(config.CLASSES), activation="softmax")(headModel)
# place the head FC model on top of the base model (this will become
# the actual model we will train)
model = Model(inputs=baseModel.input, outputs=headModel)
# loop over all layers in the base model and freeze them so they will
# *not* be updated during the first training process
for layer in baseModel.layers:
layer.trainable = False
# compile our model (this needs to be done after our setting our
# layers to being non-trainable
print("[INFO] compiling model...")
def prepare_model(label, tr_date, ts_date, x_train, x_test, y_train, y_test):
#tr_date, ts_date, x_train, x_test, y_train, y_test = prepare_data()
ts_date = timestamp_to_dataconv(ts_date)
print(x_train.shape, y_train.shape, y_test.shape, y_test.shape)
print(x_train)
'''
print(len(X_train),len(Y_train),len(X_test),len(Y_test))
'''
try:
final_model = Sequential()
final_model.add(Dense(512, input_shape=(len(x_train[0]), )))
#final_model.add(Dense(11))
final_model.add(Dense(1))
# summary of model
print("summaryof model: ", final_model.summary())
# plot graph
# plot_model(final_model, to_file="DeepNeuralnetwork.png")
opt = Adam(lr=0.001)
final_model.compile(optimizer=opt, loss=losses.logcosh)
# fit the model
final_model.fit(x_train, y_train, epochs=5, batch_size=256,
verbose=0) # validation_data=(X_test, Y_test)
final_model.evaluate(x_test, y_test, batch_size=256, verbose=0)
print("fit end")
#print("weights: ", final_model.get_weights())
#print("param: ",final_model.count_params())
#print(final_model.__reduce__())
#print(final_model.legacy_get_config())
pred = final_model.predict(x_test)
except:
print("something is wrong in model")
predicted = pred
predicted = predicted.ravel()
original = y_test
# actual converted values after model
minimum, maximum = min_max(closep1)
pred1 = calculate_actual_values(predicted, maximum, minimum)
actual1 = calculate_actual_values(original, maximum, minimum)
#print("prediii: ", len(pred1), "actual: ", len(actual1))
# actual values
mseA = mean_squared_error(actual1, pred1)
rmsefA = root_mean_square_error_fun(mseA)
maefA = mean_absolute_error_fun(actual1, pred1)
mape1A = mean_absolute_percentage_error_fun(actual1, pred1)
r_scoreA = r2_score(actual1, pred1)
print("mse: ", mseA)
print("rmse: ", rmsefA)
print("mae: ", maefA)
print("mape: ", mape1A)
print("r2score: ", r_scoreA)
#print("predicted", predicted, "actual", original)
#print("predicted1: ", pred1, "actual1: ", actual1)
errors_label = ('mse', 'rmse', 'mae', 'mape')
y_pos = np.arange(len(errors_label))
error_values = np.array([mseA, rmsefA, maefA, mape1A])
width = 0.75
tests = [
'Layer: 1', 'neurons: {512}', 'activation: {sigmoid,linear}',
'lr: 0.001'
]
plt.figure(1)
plt.subplot(211)
# plt.subplot(221)
plt.xticks(rotation=30)
plt.plot(ts_date, actual1, label="actual1", color='green')
plt.plot(ts_date, pred1, label='predicted1', color='red')
plt.grid(True)
# plt.plot(Y_train,label ='y_train',color='yellow')
# plt.plot(Y_test, label='y_test', color='pink')
plt.title("Bitcoin Price (FeedFNN) inputs " + str(label),
fontweight='bold')
plt.legend()
plt.xlabel("Time(s)")
plt.ylabel("Price of Bitcoin")
plt.subplots_adjust(hspace=0.4, wspace=0.4)
plt.subplot(223)
plt.bar(
y_pos,
error_values,
width,
align='center',
alpha=0.5,
color='red',
)
plt.xticks(y_pos, errors_label)
for a, b in zip(y_pos, error_values):
plt.text(a, b, str(b))
# plt.annotate(str(b),y_poserror_values=(a,b))
plt.title('Evaluation Criteria', fontweight='bold')
plt.xlabel('Errors')
plt.ylabel('Values')
plt.subplot(224)
# plt.subplot(222)
# plt.subplot(212)
plt.title("Architecture", fontsize=14)
# plt.text(.2,.6,'activation function')
plt.yticks(np.arange(len(tests)) * -1)
for i, s in enumerate(tests):
plt.text(0.1,
-i / 2,
s,
fontsize=12,
horizontalalignment='left',
backgroundcolor='palegreen',
wrap=True)
plt.subplots_adjust(hspace=0.4, wspace=0.4)
plt.show()
#plt.pause(0.01)
#time.sleep(0.5)
plt.close()
model = Sequential()
model.add(Conv2D(32,nb_filter1,input_shape=(48,48,1)))
model.add(Conv2D(32,nb_filter1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size = pool_size1))
model.add(Dropout(0.25))
model.add(Conv2D(64,nb_filter1))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size = pool_size1))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1028))
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',optimizer="adam",metrics=['accuracy'])
train_x=train_x.reshape(train_x.shape[0],48,48,1)
validation_x = validation_x.reshape(validation_x.shape[0], 48, 48, 1)
#datagen.fit(train_x)
callback = EarlyStopping(monitor = 'val_loss', min_delta = 0.00, patience = 1)
#model.fit_generator(datagen.flow(train_x, train_y, batch_size = batch_size), steps_per_epoch= train_x.shape[0]/batch_size, callbacks = [callback], validation_data = (validation_x, validation_y))
model.fit(train_x,train_y,batch_size=batch_size,epochs=epochs, shuffle = True, callbacks = [callback], validation_data = (validation_x, validation_y))
for layer in model.layers[:3]:
# X_means = np.mean(X_train, axis=0)
# X_stds = np.std(X_train, axis=0)
# X_train = (X_train - X_means)/(X_stds+1e-6)
# X_test = (X_test - X_means)/(X_stds+1e-6)
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# lambda shape: forget_bias*one(shape, name=None)
print('Compare to LSTM...')
model = Sequential()
model.add(LSTM(hidden_units, input_shape=X_train.shape[1:]))
# model.add(LSTM(hidden_units, input_shape=X_train.shape[1:], inner_init='glorot_uniform', forget_bias_init='one', activation='tanh', inner_activation='sigmoid'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# rmsprop = RMSprop( clipnorm=clip_norm)
# model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy'])
# checkpointer = ModelCheckpoint(filepath="/scratch/pponte/data/rnn-mist/lstm-weights"+"-bs-"+str(batch_size)+"-hu-"+str(hidden_units)+"-lr-"+str(learning_rate)+"-rho-"+str(rho)+"-clip-"+str(clip_norm)+"-epoch-{epoch:02d}-val-{val_acc:.2f}"+".hdf5", verbose=1, save_best_only=True)
# earlystopper = EarlyStopping(monitor='val_acc', patience=15, verbose=1, mode='max')
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epochs,
verbose=1,
def main():
start_time = time.time()
signal.signal(signal.SIGINT, InterruptHandler)
#signal.signal(signal.SIGKILL, InterruptHandler)
signal.signal(signal.SIGTERM, InterruptHandler)
parser = argparse.ArgumentParser(
prog='trainLSTM_MLP.py',
description='Train LSTM-MLP model for visual question answering')
parser.add_argument('--mlp-hidden-units',
type=int,
default=1024,
metavar='<mlp-hidden-units>')
parser.add_argument('--mlp-hidden-layers',
type=int,
default=3,
metavar='<mlp-hidden-layers>')
parser.add_argument('--dropout',
type=float,
default=0.5,
metavar='<dropout-rate>')
parser.add_argument('--mlp-activation',
type=str,
default='relu',
metavar='<activation-function>')
parser.add_argument('--num-epochs',
type=int,
default=100,
metavar='<num-epochs>')
parser.add_argument('--model-save-interval',
type=int,
default=5,
metavar='<interval>')
parser.add_argument('--batch-size',
type=int,
default=128,
metavar='<batch-size>')
args = parser.parse_args()
word_vec_dim = 300
img_dim = 4096
######################
# Load Data #
######################
print('Loading data...')
train_id_pairs, train_image_ids = LoadIds('train')
dev_id_pairs, dev_image_ids = LoadIds('dev')
train_questions = LoadQuestions('train')
dev_questions = LoadQuestions('dev')
train_choices = LoadChoices('train')
dev_choices = LoadChoices('dev')
train_answers = LoadAnswers('train')
dev_answers = LoadAnswers('dev')
print('Finished loading data.')
print('Time: %f s' % (time.time() - start_time))
print('-' * 100, file=sys.stderr)
print('Training Information', file=sys.stderr)
print('# of MLP hidden units: %i' % args.mlp_hidden_units, file=sys.stderr)
print('# of MLP hidden layers: %i' % args.mlp_hidden_layers,
file=sys.stderr)
print('Dropout: %f' % args.dropout, file=sys.stderr)
print('MLP activation function: %s' % args.mlp_activation, file=sys.stderr)
print('# of training epochs: %i' % args.num_epochs, file=sys.stderr)
print('Batch size: %i' % args.batch_size, file=sys.stderr)
print('# of train questions: %i' % len(train_questions), file=sys.stderr)
print('# of dev questions: %i' % len(dev_questions), file=sys.stderr)
print('-' * 100, file=sys.stderr)
######################
# Model Descriptions #
######################
# MLP model
model = Sequential()
model.add(
Dense(output_dim=args.mlp_hidden_units,
input_dim=(word_vec_dim + img_dim)))
model.add(Activation(args.mlp_activation))
model.add(Dropout(args.dropout))
for i in range(args.mlp_hidden_layers - 1):
model.add(Dense(args.mlp_hidden_units))
model.add(Activation(args.mlp_activation))
model.add(Dropout(args.dropout))
model.add(Dense(word_vec_dim))
model.add(Activation('softmax'))
json_string = model.to_json()
model_filename = 'models/mlp_units_%i_layers_%i' % (args.mlp_hidden_units,
args.mlp_hidden_layers)
open(model_filename + '.json', 'w').write(json_string)
# loss and optimizer
model.compile(loss='categorical_crossentropy', optimizer='adagrad')
print('Compilation finished.')
print('Time: %f s' % (time.time() - start_time))
########################################
# Load CNN Features and Word Vectors #
########################################
# load VGG features
print('Loading VGG features...')
VGG_features, img_map = LoadVGGFeatures()
print('VGG features loaded')
print('Time: %f s' % (time.time() - start_time))
# load GloVe vectors
print('Loading GloVe vectors...')
word_embedding, word_map = LoadGloVe()
print('GloVe vectors loaded')
print('Time: %f s' % (time.time() - start_time))
######################
# Make Batches #
######################
print('Making batches...')
# training batches
train_question_batches = [
b for b in MakeBatches(
train_questions, args.batch_size, fillvalue=train_questions[-1])
]
train_answer_batches = [
b for b in MakeBatches(train_answers['toks'],
args.batch_size,
fillvalue=train_answers['toks'][-1])
]
train_image_batches = [
b for b in MakeBatches(
train_image_ids, args.batch_size, fillvalue=train_image_ids[-1])
]
train_indices = list(range(len(train_question_batches)))
# validation batches
dev_question_batches = [
b for b in MakeBatches(
dev_questions, args.batch_size, fillvalue=dev_questions[-1])
]
dev_answer_batches = [
b for b in MakeBatches(dev_answers['labs'],
args.batch_size,
fillvalue=dev_answers['labs'][-1])
]
dev_choice_batches = [
b for b in MakeBatches(
dev_choices, args.batch_size, fillvalue=dev_choices[-1])
]
dev_image_batches = [
b for b in MakeBatches(
dev_image_ids, args.batch_size, fillvalue=dev_image_ids[-1])
]
print('Finished making batches.')
print('Time: %f s' % (time.time() - start_time))
######################
# Training #
######################
dev_accs = []
max_acc = -1
max_acc_epoch = -1
print('Training started...')
for k in range(args.num_epochs):
print('Epoch %i' % (k + 1), file=sys.stderr)
print('-' * 80)
print('Epoch %i' % (k + 1))
progbar = generic_utils.Progbar(len(train_indices) * args.batch_size)
# shuffle batch indices
random.shuffle(train_indices)
for i in train_indices:
X_question_batch = GetQuestionsMatrix(train_question_batches[i],
word_embedding, word_map)
X_image_batch = GetImagesMatrix(train_image_batches[i], img_map,
VGG_features)
Y_answer_batch = GetAnswersMatrix(train_answer_batches[i],
word_embedding, word_map)
loss = model.train_on_batch([X_question_batch, X_image_batch],
Y_answer_batch)
loss = loss[0].tolist()
progbar.add(args.batch_size, values=[('train loss', loss)])
if k % args.model_save_interval == 0:
model.save_weights(model_filename +
'_epoch_{:03d}.hdf5'.format(k + 1),
overwrite=True)
# evaluate on dev set
widgets = [
'Evaluating ',
Percentage(), ' ',
Bar(marker='#', left='[', right=']'), ' ',
ETA()
]
pbar = ProgressBar(widgets=widgets, redirect_stdout=True)
dev_correct = 0
for i in pbar(range(len(dev_question_batches))):
# feed forward
X_question_batch = GetQuestionsMatrix(dev_question_batches[i],
word_embedding, word_map)
X_image_batch = GetImagesMatrix(dev_image_batches[i], img_map,
VGG_features)
prob = model.predict_proba([X_question_batch, X_image_batch],
args.batch_size,
verbose=0)
# get word vecs of choices
choice_feats = GetChoicesTensor(dev_choice_batches[i],
word_embedding, word_map)
similarity = np.zeros((5, args.batch_size), float)
# calculate cosine distances
for j in range(5):
similarity[j] = np.diag(
cosine_similarity(prob, choice_feats[j]))
# take argmax of cosine distances
pred = np.argmax(similarity, axis=0) + 1
dev_correct += np.count_nonzero(dev_answer_batches[i] == pred)
dev_acc = float(dev_correct) / len(dev_questions)
dev_accs.append(dev_acc)
print('Validation Accuracy: %f' % dev_acc)
print('Validation Accuracy: %f' % dev_acc, file=sys.stderr)
print('Time: %f s' % (time.time() - start_time))
print('Time: %f s' % (time.time() - start_time), file=sys.stderr)
if dev_acc > max_acc:
max_acc = dev_acc
max_acc_epoch = k
model.save_weights(model_filename + '_best.hdf5', overwrite=True)
model.save_weights(model_filename + '_epoch_{:03d}.hdf5'.format(k + 1))
print(dev_accs, file=sys.stderr)
print('Best validation accuracy: epoch#%i' % max_acc_epoch)
print('Training finished.')
print('Training finished.', file=sys.stderr)
print('Time: %f s' % (time.time() - start_time))
print('Time: %f s' % (time.time() - start_time), file=sys.stderr)
def densenet121_model(img_rows,
img_cols,
color_type=1,
nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.5,
dropout_rate=0.0,
weight_decay=1e-4,
num_classes=None,
pretrained=False,
preprocess_layer=None):
'''
DenseNet 121 Model for Keras
Model Schema is based on
https://github.com/flyyufelix/DenseNet-Keras
ImageNet Pretrained Weights
Theano: https://drive.google.com/open?id=0Byy2AcGyEVxfMlRYb3YzV210VzQ
TensorFlow: https://drive.google.com/open?id=0Byy2AcGyEVxfSTA4SHJVOHNuTXc
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(img_rows, img_cols, color_type), name='data')
else:
concat_axis = 1
img_input = Input(shape=(color_type, img_rows, img_cols), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
if preprocess_layer:
x = preprocess_layer(img_input)
else:
x = img_input
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(x)
x = Convolution2D(nb_filter,
7,
7,
subsample=(2, 2),
name='conv1',
bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x_fc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x_fc = Dense(1000, name='fc6')(x_fc)
x_fc = Activation('softmax', name='prob')(x_fc)
model = Model(img_input, x_fc, name='densenet')
if pretrained:
print('Loading Imagenet weights.')
model.load_weights('data/weights/densenet121_weights_tf.h5',
by_name=True)
# if K.image_dim_ordering() == 'th':
# # Use pre-trained weights for Theano backend
# weights_path = 'imagenet_models/densenet121_weights_th.h5'
# else:
# # Use pre-trained weights for Tensorflow backend
# weights_path = 'imagenet_models/densenet121_weights_tf.h5'
# model.load_weights(weights_path, by_name=True)
# Truncate and replace softmax layer for transfer learning
# Cannot use model.layers.pop() since model is not of Sequential() type
# The method below works since pre-trained weights are stored in layers but not in the model
x_newfc = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
# x_newfc = Dense(num_classes, name='fc6')(x_newfc)
# x_newfc = Activation('softmax', name='prob')(x_newfc)
model = Model(img_input, x_newfc)
# # Learning rate is changed to 0.001
# sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
# model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
return model
nb_epochs = 1000
batch_size = 128
epsilon = .8
gamma = .8
# Recipe of deep reinforcement learning model
model = Sequential()
model.add(
Convolution2D(16,
nb_row=3,
nb_col=3,
input_shape=(1, GRID_SIZE, GRID_SIZE),
activation='relu'))
model.add(Convolution2D(16, nb_row=3, nb_col=3, activation='relu'))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(3))
model.compile(RMSprop(), 'MSE')
exp_replay = experience_replay(batch_size)
exp_replay.next() # Start experience-replay coroutine
for i in xrange(nb_epochs):
ep = episode()
S, won = ep.next() # Start coroutine of single entire episode
loss = 0.
try:
while True:
action = np.random.randint(-1, 2)
if np.random.random() > epsilon:
# Get the index of the maximum q-value of the model.
def VGG16(include_top=True,
weights=None,
input_tensor=None,
input_shape=None,
pooling=None,
nfcfilters=4096,
nconvfilters=[64, 128, 512, 512, 512],
classes=1,
nblocks=5):
seq_input = Input(shape=input_shape)
if nblocks >= 1:
# Block 1
x = Conv2D(nconvfilters[0], (4, 3),
activation='relu',
padding='same',
name='block1_conv1')(seq_input)
x = Conv2D(nconvfilters[0], (1, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
x = MaxPooling2D((1, 2), name='block1_pool')(x)
if nblocks >= 2:
# Block 2
x = Conv2D(nconvfilters[1], (1, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = Conv2D(nconvfilters[1], (1, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
x = MaxPooling2D((1, 2), name='block2_pool')(x)
if nblocks >= 3:
# Block 3
x = Conv2D(nconvfilters[2], (1, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = Conv2D(nconvfilters[2], (1, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = Conv2D(nconvfilters[2], (1, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
x = MaxPooling2D((1, 2), name='block3_pool')(x)
if nblocks >= 4:
# Block 4
x = Conv2D(nconvfilters[3], (1, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = Conv2D(nconvfilters[3], (1, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = Conv2D(nconvfilters[3], (1, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
x = MaxPooling2D((1, 2), name='block4_pool')(x)
if nblocks >= 5:
# Block 5
x = Conv2D(nconvfilters[4], (1, 3),
activation='relu',
padding='same',
name='block5_conv1')(x)
x = Conv2D(nconvfilters[4], (1, 3),
activation='relu',
padding='same',
name='block5_conv2')(x)
x = Conv2D(nconvfilters[4], (1, 3),
activation='relu',
padding='same',
name='block5_conv3')(x)
x = MaxPooling2D((1, 2), name='block5_pool')(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(nfcfilters, activation='relu', name='fc1')(x)
x = Dense(nfcfilters, activation='relu', name='fc2')(x)
if classes == 1:
x = Dense(classes, activation='sigmoid', name='predictions')(x)
else:
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# inputs
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = seq_input
# Create model
model = Model(inputs, x, name='vgg16')
if weights:
raise NotImplementedError
return model
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LEN,
trainable=False)
str_now = str(datetime.datetime.now()).replace('-','').replace(' ','_').replace(':','')
saved_model_name = './DeepLearning_model_h5_file/jifujine_model_%s.h5'%str_now[:15]
print('Training model.')
sequence_input = Input(shape=(MAX_SEQUENCE_LEN,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = LSTM(128)(embedded_sequences)
x = Dense(64, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(64, activation='relu')(x)
x = Dense(64, activation='relu')(x)
preds = Dense(y.shape[1], activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
# train section
# model.fit(data, y, batch_size=32, epochs=20, validation_split=0.1)
model.fit(data, y, batch_size=32, epochs=30)
print('Complete training, and now saving model to %s\n'%saved_model_name)
model.save(saved_model_name)
X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
history = model.fit(X_train, Y_train,
target_tags = np.zeros((number_of_target_examples, 1))
non_target_tags = np.ones((number_of_non_target_examples, 1))
all_tags = np.vstack((target_tags, non_target_tags))
Y_train = all_tags
all_samples = np.vstack((all_target_metrix, all_non_target_metrix))
print(all_target_metrix.shape)
#currently not choosed randomally
#x_row = all_target_metrix[0:50,:]
x = all_samples.reshape(all_samples.shape[0], 55, 1, 80)
model = Sequential()
model.add(Convolution2D(200, 55, 1, 15, border_mode='full'))
model.add(Activation('relu'))
model.add(MaxPooling2D(poolsize=(1, 2)))
#200x1x40
model.add(Flatten())
model.add(Dense(200 * (80 + 14) / 2, 55))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(55, 1))
model.add(Activation('softmax'))
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer=sgd)
model.fit(x, Y_train, batch_size=32, nb_epoch=100, show_accuracy=True)
import pandas
from pathlib import Path
from keras.models import Sequential
from keras.layers.core import Dense, Activation
# load dataset
from sklearn.model_selection import train_test_split
import pandas as pd
dataset = pd.read_csv(Path("Breas Cancer.csv"))
# print(dataset)
dataset["diagnosis"]= dataset["diagnosis"].replace("M",0)
dataset["diagnosis"]= dataset["diagnosis"].replace("B",1)
import numpy as np
X_train, X_test, Y_train, Y_test = train_test_split(dataset.iloc[:,2:32], dataset.iloc[:,1],
test_size=0.25, random_state=30)
# print(dataset.shape)
np.random.seed(155)
my_first_nn = Sequential() # create model
my_first_nn.add(Dense(40, input_dim=30, activation='relu')) # hidden layer
my_first_nn.add(Dense(29, input_dim=40, activation='relu'))
# my_first_nn.add(Dense(8, input_dim=20, activation='relu'))
my_first_nn.add(Dense(1, input_dim=29,activation='sigmoid')) # output layer
my_first_nn.compile(loss='binary_crossentropy', optimizer='adam')
my_first_nn_fitted = my_first_nn.fit(X_train, Y_train, epochs=100, verbose=0,
initial_epoch=0)
print(my_first_nn.summary())
print(my_first_nn.evaluate(X_test, Y_test, verbose=0))
data = np.loadtxt("bigdata.dat")
print "Data Imported"
nz = (data[:,0]==-1)
data = data[nz==0,:]
ny = (data[:,0]==25)
data = data[ny==0,:]
X_train = data[1:100000,1:31].astype('float32')
labels1 = data[1:100000,0].astype('int32')
y_train = np_utils.to_categorical(labels1)
X_test = data[100001:128990,1:31].astype('float32')
labels2 = data[100001:128990,0].astype('int32')
y_test = np_utils.to_categorical(labels2)
# Creating Model
model = Sequential()
model.add(Dense(200, input_dim=30,init="uniform"))
model.add(Activation('relu'))
#model.add(Dropout(0.25))
model.add(Dense(200,init="uniform"))
model.add(Activation('relu'))
#model.add(Dropout(0.25))
model.add(Dense(25,init="uniform"))
model.add(Activation('softmax'))
#sgd = SGD(lr=0.05, decay=1e-6, momentum=0.9)
model.compile(loss='categorical_crossentropy',optimizer='RMSprop')
model.fit(X_train, y_train, batch_size=200, nb_epoch=20,verbose=1,show_accuracy=True,validation_data=(X_test,y_test))
# Testing model output
classes = model.predict_classes(data[:,1:31], batch_size=32)
proba = model.predict_proba(data[:,1:31], batch_size=32)
model_name = args.model
# model definition - NVidia architecture
model = Sequential()
# cropping
model.add(
Cropping2D(cropping=(cropping_tb, cropping_lr), input_shape=image_size))
# normalizing image ( to [-0.5, 0.5] )
model.add(Lambda(lambda x: (x / 255.0) - 0.5))
model.add(Conv2D(24, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(36, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(48, (5, 5), strides=(2, 2), activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(10))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam')
steps_train = int(math.ceil(num_train_samples / batch_size))
steps_valid = int(math.ceil(num_valid_samples / batch_size))
train_generator = generator(train_samples, batch_size)
valid_generator = generator(valid_samples, batch_size)
# train model
print("Start training model {}".format(model_name))
history = model.fit_generator(train_generator,
steps_train,
validation_data=valid_generator,
'NER_Dataset/train_bio_new_lowercase.txt', 0)
X_train = X_train.reshape(-1, maxlen * 100)
print("X_train", X_train.shape)
print("y_train", y_train.shape)
max_features = len(word2ind)
print("max_features", max_features)
out_size = len(label2ind) + 1
print("test data is loading ...")
X_test, y_test, _, _, _ = data_loader('NER_Dataset/test_bio_new_lowercase.txt',
maxlen)
X_test = X_test.reshape(-1, maxlen * 100)
model = Sequential()
model.add(Embedding(8, 8, input_length=maxlen * 100, mask_zero=False))
model.add(TimeDistributed(Dense(8, input_shape=(None, 100))))
model.add(Flatten())
model.add(Dense(maxlen))
model.add(Reshape((maxlen, 1)))
model.add(
Bidirectional(
LSTM(32, return_sequences=True, dropout=0.5, recurrent_dropout=0.25)))
model.add(TimeDistributed(Dense(out_size)))
model.add(Activation('softmax'))
adam = optimizers.Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
model.compile(loss='categorical_crossentropy',
train_y = train_y.values
test_X = test_X.values
test_y = test_y.values
train_X = train_X.reshape(train_X.shape[0], train_X.shape[1], 1)
test_X = test_X.reshape(test_X.shape[0], test_X.shape[1], 1)
# Define the LSTM model
model = Sequential()
model.add(
LSTM(input_shape=(window_size, 1),
output_dim=window_size,
return_sequences=True))
model.add(Dropout(0.5))
model.add(LSTM(256))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation("linear"))
model.compile(loss="mse", optimizer="adam")
model.summary()
# Train
start = time.time()
model.fit(train_X,
train_y,
batch_size=window_size,
epochs=epoch,
validation_split=0.1)
print("> Compilation Time : ", time.time() - start)
def moving_test_window_preds(n_future_preds):
''' n_future_preds - Represents the number of future predictions we want to make
This coincides with the number of windows that we will move forward
# 不太清楚.add方法的实现，只能直接调用方法
# 构造神经网络，按照典型的卷积神经网络进行两次卷积、池化，并一维序列化，全连接，输出
model = Sequential()
# 第一次卷积、池化，第一层要使用input_shape参数，此处表示20*20的灰度图
model.add(
Conv2D(20, (5, 5),
padding="same",
input_shape=(20, 20, 1),
activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# 第二次卷积、池化
model.add(Conv2D(50, (5, 5), padding="same", activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# 全连接隐含层的结点设定为500个
model.add(Flatten())
model.add(Dense(500, activation="relu"))
# 输出层设定有32个结点，正好对应需要识别的每一个字母或数字
model.add(Dense(32, activation="softmax"))
# 编译模型，标准步骤之一，不清楚用途
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# 训练神经网络，validation_data设定验证集，每批32个数据，总共10轮
model.fit(X_train,
Y_train,
validation_data=(X_test, Y_test),
batch_size=32,
epochs=10,
verbose=1)
# 保存模型(使用模型自带函数)
model.save(MODEL_FILENAME)
def build(numChannels, imgRows, imgCols, numClasses, activation="relu", kernel_initializer="he_normal",
weightsPath=None):
num_filter = 64
inputShape = (numChannels, imgRows, imgCols)
if K.image_data_format() == 'channels_last':
inputShape = (imgRows, imgCols, numChannels)
model = Sequential()
# Block 1
model.add(Conv2D(num_filter, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer,
input_shape=inputShape))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.5))
# Block 2
model.add(Conv2D(num_filter * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.5))
# Block 3
model.add(Conv2D(num_filter * 2 * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2, kernel_size=(3, 3), padding="same", kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.5))
# Block 4
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.5))
# Block 5
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Conv2D(num_filter * 2 * 2 * 2, kernel_size=(3, 3), padding="same",
kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dropout(0.5))
# Classifier
model.add(Flatten())
model.add(Dense(1024, kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Dropout(0.5))
model.add(Dense(512, kernel_initializer=kernel_initializer))
model.add(BatchNormalization())
model.add(Activation(activation=activation))
model.add(Dropout(0.5))
model.add(Dense(numClasses, activation='softmax', kernel_initializer=kernel_initializer))
if weightsPath is not None:
print("[INFO] load model...")
model.load_weights(weightsPath)
return model
def DenseNet(nb_dense_block=4,
growth_rate=32,
nb_filter=64,
reduction=0.0,
dropout_rate=0.0,
weight_decay=1e-4,
classes=1000,
weights_path=None):
'''Instantiate the DenseNet 121 architecture,
# Arguments
nb_dense_block: number of dense blocks to add to end
growth_rate: number of filters to add per dense block
nb_filter: initial number of filters
reduction: reduction factor of transition blocks.
dropout_rate: dropout rate
weight_decay: weight decay factor
classes: optional number of classes to classify images
weights_path: path to pre-trained weights
# Returns
A Keras model instance.
'''
eps = 1.1e-5
# compute compression factor
compression = 1.0 - reduction
# Handle Dimension Ordering for different backends
global concat_axis
if K.image_dim_ordering() == 'tf':
concat_axis = 3
img_input = Input(shape=(224, 224, 3), name='data')
else:
concat_axis = 1
img_input = Input(shape=(3, 224, 224), name='data')
# From architecture for ImageNet (Table 1 in the paper)
nb_filter = 64
nb_layers = [6, 12, 24, 16] # For DenseNet-121
# Initial convolution
x = ZeroPadding2D((3, 3), name='conv1_zeropadding')(img_input)
x = Convolution2D(nb_filter,
7,
7,
subsample=(2, 2),
name='conv1',
bias=False)(x)
x = BatchNormalization(epsilon=eps, axis=concat_axis, name='conv1_bn')(x)
x = Scale(axis=concat_axis, name='conv1_scale')(x)
x = Activation('relu', name='relu1')(x)
x = ZeroPadding2D((1, 1), name='pool1_zeropadding')(x)
x = MaxPooling2D((3, 3), strides=(2, 2), name='pool1')(x)
# Add dense blocks
for block_idx in range(nb_dense_block - 1):
stage = block_idx + 2
x, nb_filter = dense_block(x,
stage,
nb_layers[block_idx],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
# Add transition_block
x = transition_block(x,
stage,
nb_filter,
compression=compression,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
nb_filter = int(nb_filter * compression)
final_stage = stage + 1
x, nb_filter = dense_block(x,
final_stage,
nb_layers[-1],
nb_filter,
growth_rate,
dropout_rate=dropout_rate,
weight_decay=weight_decay)
x = BatchNormalization(epsilon=eps,
axis=concat_axis,
name='conv' + str(final_stage) + '_blk_bn')(x)
x = Scale(axis=concat_axis,
name='conv' + str(final_stage) + '_blk_scale')(x)
x = Activation('relu', name='relu' + str(final_stage) + '_blk')(x)
x = GlobalAveragePooling2D(name='pool' + str(final_stage))(x)
x = Dense(classes, name='fc6')(x)
x = Activation('softmax', name='prob')(x)
model = Model(img_input, x, name='densenet')
if weights_path is not None:
model.load_weights(weights_path)
return model
HIDDEN_DIM = args['hidden_dim']
SEQ_LENGTH = args['seq_length']
WEIGHTS = args['weights']
GENERATE_LENGTH = args['generate_length']
LAYER_NUM = args['layer_num']
# Creating training data
X, y, VOCAB_SIZE, ix_to_char = load_data(DATA_DIR, SEQ_LENGTH)
# Creating and compiling the Network
model = Sequential()
model.add(LSTM(HIDDEN_DIM, input_shape=(None, VOCAB_SIZE), return_sequences=True))
for i in range(LAYER_NUM - 1):
model.add(LSTM(HIDDEN_DIM, return_sequences=True))
model.add(TimeDistributed(Dense(VOCAB_SIZE)))
model.add(Activation('softmax'))
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
# Generate some sample before training to know how bad it is!
generate_text(model, args['generate_length'], VOCAB_SIZE, ix_to_char)
if not WEIGHTS == '':
model.load_weights(WEIGHTS)
nb_epoch = int(WEIGHTS[WEIGHTS.rfind('_') + 1:WEIGHTS.find('.')])
else:
nb_epoch = 0
# Training if there is no trained weights specified
if args['mode'] == 'train' or WEIGHTS == '':
while True:
model.add(
Convolution2D(nb_filters * 2,
nb_conv,
nb_conv,
border_mode='valid',
input_shape=(1, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters * 2, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))
c = 3.5
Weight_Decay = c / float(X_train.shape[0])
model.add(Flatten())
model.add(Dense(128, W_regularizer=l2(Weight_Decay)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
hist = model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
verbose=1,
validation_data=(X_valid, Y_valid))
Train_Result_Optimizer = hist.history
Train_Loss = np.asarray(Train_Result_Optimizer.get('loss'))
