def build_model(mode=None, maxlen=None, output_dim=None):
print('Build model...')
def _scheduler(epoch):
global counter
counter += 1
rate = learning_rates[counter]
#model.lr.set_value(rate)
print(model.optimizer.lr)
#print(counter, rate)
return model.optimizer.lr
change_lr = LRS(_scheduler)
model = Sequential()
model.add(LSTM(128 * 15, return_sequences=False,
input_shape=(maxlen, 512)))
model.add(Dropout(0.5))
model.add(Dense(output_dim))
model.add(Activation('softmax'))
if mode == "rms":
optimizer = RMSprop(lr=0.01)
if mode == "adam":
optimizer = Adam()
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model, change_lr
def set_tr_params(args):
#### OPTIMIZER
if (args.optim == "sgd"):
print("Using SGD")
opt = SGD(lr=args.lr, decay=1e-6, momentum=0.9)
elif (args.optim == "adam"):
print("Using Adam")
opt = Adam(lr=args.lr,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
elif (args.optim == "rmsprop"):
print("Using RMSprop")
opt = RMSprop(lr=args.lr, rho=0.9, epsilon=None, decay=0.0)
## LR Scheduller
if (args.lra == True):
e1 = int(args.epochs * 0.5)
e2 = int(args.epochs * 0.75)
print("Learning rate annealing epochs: 0-->", e1, "-->", e2, "-->",
args.epochs)
print("Learning rate annealing LR:", args.lr, "-->",
args.lr / args.lra_scale, "-->",
args.lr / (args.lra_scale * args.lra_scale))
def scheduler(epoch):
if epoch < e1:
return args.lr
elif epoch < e2:
if (epoch == e1):
print("===============================")
print("New learning rate:", args.lr / args.lra_scale)
print("===============================")
return args.lr / args.lra_scale
else:
if (epoch == e2):
print("===============================")
print("New learning rate:",
args.lr / (args.lra_scale * args.lra_scale))
print("===============================")
return args.lr / (args.lra_scale * args.lra_scale)
set_lr = LRS(scheduler)
callbacks = [set_lr]
else:
print("Learning rate=", args.lr, "no annealing")
callbacks = []
## Model checkpoint
if (args.save_epochs):
filepath = args.save_model + "_{epoch:03d}.h5"
checkpoint = ModelCheckpoint(filepath, verbose=1, save_best_only=False)
callbacks.append(checkpoint)
return opt, callbacks
opt = SGD(lr=0.1, decay=1e-6)
modelFinal.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.001),
metrics=['accuracy'])
# DEFINE A LEARNING RATE SCHEDULER
def scheduler(epoch):
if epoch < 25:
return .001
elif epoch < 50:
return 0.0001
else:
return 0.00001
set_lr = LRS(scheduler)
## TRAINING
history = modelFinal.fit_generator(
datagen.flow(x_train, y_train, batch_size=batch_size),
steps_per_epoch=len(x_train) / batch_size,
epochs=epochs,
validation_data=testdatagen.flow(
x_test, y_test), # uncomment this for simpler data aug
validation_steps=len(x_test) /
batch_size, # uncomment this for simpler data aug
#validation_data=(x_test, y_test), # comment this for simpler data aug
callbacks=[set_lr])
def define_model(x):
global st
global modname
global model
global a
st += 1
n_conv = int(x[0])
nla = int(x[1])
np = int(x[2])
np1 = int(x[3])
np2 = int(x[4])
drop = x[5]
a = x[6]
def decay(ep):
global a
lr = a/((ep)+1)#simple 1 param 1/(t) decay
return lr
lr = LRS(decay)
#input layers
pin = Input(shape=(num_wave,num_ang))
din = Input(shape=(num_wave,num_ang))
#variable number of convolutional layers
#independent layers for psi and delta
convrp = Conv1D(64,4,activation='relu',padding='same')(pin)
convrp = MaxPooling1D(pool_size = 2)(convrp)
for i in range(n_conv):
convrp = Conv1D(64,4,activation='relu')(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
r = Flatten()(convrp)
convrs = Conv1D(64,4,activation='relu',padding='same')(din)
convrs = MaxPooling1D(pool_size = 2)(convrs)
for i in range(n_conv):
convrs = Conv1D(64,4,activation='relu')(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
t = Flatten()(convrs)
#pass the convolutoinal layers through dense layers, still indepent by spectra type
r = Dense(np,activation='relu')(r)
r = Dropout(drop)(r)
t = Dense(np,activation='relu')(t)
t = Dropout(drop)(t)
#combine the two spectral types
a = Add()([r,t])
#pass the data through more dense layers
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
for i in range(nla):
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
#indepent layers for individual spectra in linear array format
rp = Dense(600,activation='relu')(a)
rs = Dense(600,activation='relu')(a)
tp = Dense(600,activation='relu')(a)
ts = Dense(600,activation='relu')(a)
#output nodes
rp = Dense(600,activation=None)(rp)
rs = Dense(600,activation=None)(rs)
tp = Dense(600,activation=None)(tp)
ts = Dense(600,activation=None)(ts)
#define the model
model = Model([pin,din],[rp,rs,tp,ts])
# compile the model using the adam optimizer and appropriate loss functions
#the loss weights can be used to tune the relative importance of output data .
#I find that increasing the loss weight for the thickness layer tends to improve fitting,
#since there loss fuctions are different for materials and thicknesses
model.compile(optimizer='adam',loss=['mse','mse','mse','mse'],metrics = ['mse'],loss_weights=[2,1,1,1])
model.summary()
#model name can be anything you want
modname = dr+'general_5lay4matinverse_model_v-tma_convolution_pd2rt_'+str(st)+'step_'+date.strftime("%Y")+date.strftime("%m")+date.strftime("%d")
#print the model summary to file for records, the modelname in log also helps with identificaiton of the log files
print('-'*57)
print(modname)
print('-'*57)
with open(modname+'_summary.txt', 'w') as f:
with redirect_stdout(f):
model.summary()
def define_model(x):
global st
global modname
global model
global a
st += 1
n_conv = int(x[0])
nla = int(x[1])
np = int(x[2])
np1 = int(x[3])
np2 = int(x[4])
drop = x[5]
a = x[6]
def decay(ep):
global a
lr = a/((ep)+1)#simple 1 param 1/(t) decay
return lr
lr = LRS(decay)
#input layers
rpin = Input((200,3))
rsin = Input((200,3))
tpin = Input((200,3))
tsin = Input((200,3))
#variable number of convolutional layers
#independent layers for psi and delta
convrp = Conv1D(64,4,activation='relu',padding='same')(rpin)
convrp = MaxPooling1D(pool_size = 2)(convrp)
for i in range(n_conv):
convrp = Conv1D(64,4,activation='relu')(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
frp = Flatten()(convrp)
convrs = Conv1D(64,4,activation='relu',padding='same')(rsin)
convrs = MaxPooling1D(pool_size = 2)(convrs)
for i in range(n_conv):
convrs = Conv1D(64,4,activation='relu')(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
frs = Flatten()(convrs)
convtp = Conv1D(64,4,activation='relu',padding='same')(tpin)
convtp = MaxPooling1D(pool_size = 2)(convtp)
for i in range(n_conv):
convtp = Conv1D(64,4,activation='relu')(convtp)
convtp = MaxPooling1D(pool_size = 2)(convtp)
convtp = MaxPooling1D(pool_size = 2)(convtp)
ftp = Flatten()(convtp)
convts = Conv1D(64,4,activation='relu',padding='same')(tsin)
convts = MaxPooling1D(pool_size = 2)(convts)
for i in range(n_conv):
convts = Conv1D(64,4,activation='relu')(convts)
convts = MaxPooling1D(pool_size = 2)(convts)
convts = MaxPooling1D(pool_size = 2)(convts)
fts = Flatten()(convts)
#add the two polarization states together in some dense layers
#pass the convolutoinal layers through dense layers, still indepent by spectra type
r = Add()([frp,frs])
t = Add()([ftp,fts])
r = Dense(np,activation='relu')(r)
r = Dropout(drop)(r)
t = Dense(np,activation='relu')(t)
t = Dropout(drop)(t)
#combine the two spectral types
a = Add()([r,t])
#pass the data through more dense layers
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
for i in range(nla):
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
#indepent layers for the materials and thickness funneling the data to small number of output nodes
m = Dense(np2,activation='relu')(a)
t = Dense(np2,activation='relu')(a)
m = Dense(200,activation='relu')(m)
t = Dense(200,activation='relu')(t)
m = Dense(100,activation='relu')(m)
t = Dense(50,activation='relu')(t)
#output nodes
#thickness is a number in so there is no activation
thout = Dense(2,activation=None)(t)
#materials guesses a probability array for each material
#real material is always a normalized array with 1 at the material index i.e. [0 1 0 0 0]
mout1 = Dense(5,activation='softmax')(m)
mout2 = Dense(5,activation='softmax')(m)
#define the model
model = Model([rpin,rsin,tpin,tsin],[thout,mout1,mout2])
lr= 0.001
decay= lr/n_epoch
game = Game(game_size, zit_score_to_win)
from keras.callbacks import LearningRateScheduler as LRS
import keras.backend as K
def scheduler(epoch):
if epoch% 3 == 0 and epoch != 0:
lr = K.get_value(zit_model.optimizer.lr)
K.set_value(zit_model.optimizer.lr, lr*0.1)
print('lr change to {}'.format(lr*0.1))
return K.get_value(zit_model.optimizer.lr)
reduce_lr = LRS(scheduler)
def generate_array_from_file(data_path):
global count
while 1:
load_path = os.path.join(data_path, 'new_cnn_dataset_'+str(count)+'.npz')
train_data = np.load(load_path)['arr_0']
train_label = np.load(load_path)['arr_1']
count = count+1
if count >= ite:
count = 0
train_data = train_data.reshape((train_data.shape[0],4,4,12))
yield (train_data, train_label)
#print(x_train.max())
# x_train = x_train[:, 38:]
# x_val = x_val[:, 38:]
print("x train shape")
print(x_train.shape)
x_train = x_train.reshape(x_train.shape[0], 90)
x_val = x_val.reshape(x_val.shape[0], 90)
x_train = x_train.astype('float32')
x_val = x_val.astype('float32')
# learning schedule callback
lrate = LRS(step_decay)
callbacks_list = [lrate]
#callbacks = [TensorBoard(log_dir="logs/classifier/{}".format(time()), write_graph=True)]
model = cmodel()
history = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_val, y_val),
shuffle=True,
callbacks=callbacks_list)
scores = model.evaluate(x_val, y_val, verbose=1)
print('Test loss:', scores[0])
if (delta.min() < 0.):
resc = delta + np.pi
resc = np.rad2deg(resc) / 360.
return resc
# learning rate decay function
def decay(ep):
'learning rate decay function, may include others in future'
a = .0025
lr = a / ((ep) + 1)
return lr
# can use this guy below as a callback when doing Model.fit
lr = LRS(decay)
#function to read in the data
@jit(nopython=True)
def read_dat(fname, dataset, nlay, nang, nwvl):
'''
function to read in the training data within an .h5 file and rescale
NOTE: saves rescale of delta and psi for 'rs_dat'
currently used for ENZ proj with data for angle, thicknesses, rp, rs, tp, ts, psi, delta
NOTE: data should be a [N,T] array, w/ N for num of runs and T = nang + nlay + 6*nang*nwvl
inputs:
fname - string - file name containing the training data, pls use full file name
dataset - string - name of the dataset in the .h5 file
NOTE: currently doesn't support reading in data from more than 1 dataset
nlay - integer - number of layers in the multistack
def define_model(x):
global st
global modname
global model
global a
st += 1
n_conv = int(x[0])
nla = int(x[1])
np = int(x[2])
np1 = int(x[3])
np2 = int(x[4])
drop = x[5]
a = x[6]
def decay(ep):
global a
lr = a/((ep)+1)#simple 1 param 1/(t) decay
return lr
lr = LRS(decay)
#input layers
rpin = Input((200,3))
rsin = Input((200,3))
tpin = Input((200,3))
tsin = Input((200,3))
#variable number of convolutional layers
#independent layers for psi and delta
convrp = Conv1D(64,4,activation='relu',padding='same')(rpin)
convrp = MaxPooling1D(pool_size = 2)(convrp)
for i in range(n_conv):
convrp = Conv1D(64,4,activation='relu')(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
convrp = MaxPooling1D(pool_size = 2)(convrp)
frp = Flatten()(convrp)
convrs = Conv1D(64,4,activation='relu',padding='same')(rsin)
convrs = MaxPooling1D(pool_size = 2)(convrs)
for i in range(n_conv):
convrs = Conv1D(64,4,activation='relu')(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
convrs = MaxPooling1D(pool_size = 2)(convrs)
frs = Flatten()(convrs)
convtp = Conv1D(64,4,activation='relu',padding='same')(tpin)
convtp = MaxPooling1D(pool_size = 2)(convtp)
for i in range(n_conv):
convtp = Conv1D(64,4,activation='relu')(convtp)
convtp = MaxPooling1D(pool_size = 2)(convtp)
convtp = MaxPooling1D(pool_size = 2)(convtp)
ftp = Flatten()(convtp)
convts = Conv1D(64,4,activation='relu',padding='same')(tsin)
convts = MaxPooling1D(pool_size = 2)(convts)
for i in range(n_conv):
convts = Conv1D(64,4,activation='relu')(convts)
convts = MaxPooling1D(pool_size = 2)(convts)
convts = MaxPooling1D(pool_size = 2)(convts)
fts = Flatten()(convts)
#add the two polarization states together in some dense layers
#pass the convolutoinal layers through dense layers, still indepent by spectra type
r = Add()([frp,frs])
t = Add()([ftp,fts])
r = Dense(np,activation='relu')(r)
r = Dropout(drop)(r)
t = Dense(np,activation='relu')(t)
t = Dropout(drop)(t)
#combine the two spectral types
a = Add()([r,t])
#pass the data through more dense layers
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
for i in range(nla):
a = Dense(np1,activation='relu')(a)
a = Dropout(drop)(a)
#indepent layers for the materials and thickness funneling the data to small number of output nodes
m = Dense(np2,activation='relu')(a)
t = Dense(np2,activation='relu')(a)
m = Dense(200,activation='relu')(m)
t = Dense(200,activation='relu')(t)
m = Dense(100,activation='relu')(m)
t = Dense(50,activation='relu')(t)
#output nodes
#thickness is a number in so there is no activation
thout = Dense(1,activation=None)(t)
#materials guesses a probability array for each material
#real material is always a normalized array with 1 at the material index i.e. [0 1 0 0 0]
mout1 = Dense(5,activation='softmax')(m)
#define the model
model = Model([rpin,rsin,tpin,tsin],[thout,mout1])
# compile the model using the adam optimizer and appropriate loss functions
#the loss weights can be used to tune the relative importance of output data .
#I find that increasing the loss weight for the thickness layer tends to improve fitting,
#since there loss fuctions are different for materials and thicknesses
model.compile(optimizer='adam',loss=['mse','categorical_crossentropy'],metrics = ['mse','accuracy'],loss_weights=[2,1])
model.summary()
#model name can be anything you want
modname = dr+'general_1lay4matinverse_model_v-tma_convolution_rprstpts_'+str(st)+'step_'+date.strftime("%Y")+date.strftime("%m")+date.strftime("%d")
#print the model summary to file for records, the modelname in log also helps with identificaiton of the log files
print('-'*57)
print(modname)
print('-'*57)
with open(modname+'_summary.txt', 'w') as f:
with redirect_stdout(f):
model.summary()