def ready(self):
if self.use_interface == None:
if self.initialize_icd():
info('Using Nokia ICd\n')
return
devlist = []
if self.use_interface != None:
# user specified inteface
devlist.append(self.use_interface)
else:
devlist.append('bat0')
for i in xrange(3):
devlist.append('wlan%d' %(i))
for i in xrange(9):
devlist.append('eth%d' %(i))
for dev in devlist:
(ip, bcast) = self.get_ip_address(dev)
if ip != None:
info('Using interface %s: %s %s\n' % (dev, ip, bcast))
self.interfaces[dev] = (ip, bcast)
sch = get_plugin_by_type(PLUGIN_TYPE_SCHEDULER)
sch.call_periodic(POLL_INTERVAL * sch.SECOND, self.periodic_poll)
def __init__(self, options):
global images_path, proximatedir, broadcastports
self.options = options
self.register_plugin(PLUGIN_TYPE_STATE)
self.pluginstorage = {}
home = os.getenv('HOME')
if home == None:
die('HOME not defined\n')
images_path = '%s/MyDocs/.images' %(home)
if options.proximatedir != None:
proximatedir = options.proximatedir
else:
proximatedir = '%s/.proximate' %(home)
if not xmkdir(proximatedir):
die('Can not create a proximate directory: %s\n' %(proximatedir))
set_log_file('%s/log' %(proximatedir))
read_communities()
self.config_read()
if options.broadcastports != None:
broadcastports = options.broadcastports
if options.traffic_mode != None:
set_traffic_mode(options.traffic_mode)
info('I am %s aka %s\n' %(myself.get('nick'), myself.get('uid')))
def notify(self, msg, highpri=False, delay=None):
if highpri:
info("IMPORTANT: " + msg + "\n")
else:
info(msg + "\n")
if self.ui != None:
self.ui.notification_show(msg, highpri, delay, None)
def set_status(self, msg, timeout=None):
""" progress() is used to report on-going operations.
If msg != None, report progress. Otherwise stop it.
Status is cancelled withint timeout seconds if timeout != None.
Calling set_status() or clear_status() before the timeout
is allowed.
Returns a number for pending status that can be used
to clear the status by calling self.clear_status(number).
"""
self.msg = msg
if progress_callback != None:
indicators = filter(lambda p: p.msg != None, progress_indicators)
progress_callback(indicators)
else:
info("Progress indicator: %s\n" % msg)
if self.timeouttag != None:
source_remove(self.timeouttag)
self.timeouttag = None
if msg == None:
return None
self.cnt += 1
if timeout != None:
self.timeouttag = timeout_add_seconds(timeout, self.clear_status, self.cnt)
return self.cnt
def init():
""" Bind a default and a random port.
The random port is used for local network communication.
The default port is used to establish remote connections. """
global community
community = get_plugin_by_type(PLUGIN_TYPE_COMMUNITY)
create_tcp_listener(DEFAULT_PROXIMATE_PORT, tcp_listener_accept, reuse=True)
success = False
for i in xrange(PORT_RETRIES):
port = community.get_rpc_port()
if port == DEFAULT_PROXIMATE_PORT:
continue
(rfd, tag) = create_tcp_listener(port, tcp_listener_accept, reuse=True)
if rfd != None:
info('Listening to TCP connections on port %d\n' %(port))
success = True
break
warning('Can not bind to TCP port %d\n' %(port))
# Generate a new port number so that next iteration will not fail
if not community.gen_port():
break
if not success:
warning('Can not listen to TCP connections\n')
def user_notify(self, user, msg, highpri=False, delay=None):
if highpri:
info("IMPORTANT: %s %s\n" % (user.tag(), msg))
else:
info("%s %s\n" % (user.tag(), msg))
if self.ui != None:
self.ui.notification_show("%s %s" % (user.tag(), msg), highpri, delay, user)
def create_udp_listener(self):
port = community.get_rpc_port()
info('fetcher: Listening to UDP port %d\n' % port)
rfd = create_udp_socket('', port, False, reuse = True)
if rfd == None:
warning('Can not listen to UDP broadcasts\n')
return
rfd.setblocking(False)
io_add_watch(rfd, IO_IN, self.udp_listener_read)
def __init__(self, options):
self.register_plugin(PLUGIN_TYPE_COMMUNITY)
self.register_server(TP_HELLO, Hello_Server)
self.fetcher = None
self.fetchhandlers = {
TP_HELLO: self.handle_hello,
'uprofile': self.handle_user_profile_fetch,
'iconrequest': self.handle_icon_request,
'iconpush': self.handle_icon_push,
'cprofile': self.handle_community_profile_fetch,
'cprofiles': self.handle_community_profiles_fetch,
'cinvite': self.handle_invite_community,
}
self.notify = None
self.net = None
self.community_gui = None
self.req_counter = 0
self.activeport = options.activeport
# Note ipactive is not dependent on udp_listen and udp_send variables
self.ipactive = True
self.activeusers = {}
self.remoteusers = {}
for user in self.get_users(False):
remotes = user.get('remotes')
if remotes != None and len(remotes) > 0:
self.remoteusers[user] = 0
self.myself = get_myself()
self.myuid = self.myself.get('uid')
self.udp_listen = (options.udpmode & 1) != 0
self.udp_send = (options.udpmode & 2) != 0
if not self.udp_listen or not self.udp_send:
info('UDP broadcast listen: %s send: %s\n' % (self.udp_listen, self.udp_send))
self.blacklist = {}
self.blistcom = self.create_community(BLACKLIST_COMMUNITY_NAME, peer=False, public=False)
self.blistcom.set('invisible', True)
self.iconfetchlimiters = {'user': Rate_Limiter(ICON_PUSH_INTERVAL)}
self.personal_communities = options.personal_communities
# Create a community of friends, if it doesn't already exist
friends = self.get_friend_community()
if friends == None:
friends = self.create_community(FRIENDS_COMMUNITY_NAME, peer=False, public=False, desc='My friends')
self.set_community_icon(friends, get_path(FRIEND_COMMUNITY_ICON))
def got_bye(self, d, address):
""" User quit, denounce """
validator = {
'uid': lambda s: valid_uid(s) and s != self.myuid,
}
if not validate(validator, d):
return
user = self.safe_get_user(d.get('uid'), address[0])
if user == None:
info('Rejecting quit message from uid %s\n' % d.get('uid'))
else:
self.denounce_user(user)
def got_rpc_msg(self, data, address):
if not self.ipactive:
return
d = fmt_bdecode({'t': str}, data)
if d == None:
return
if d['t'] == TP_HELLO:
self.got_hello(d, address)
elif d['t'] == TP_QUIT:
self.got_bye(d, address)
else:
info('Invalid RPC hello type: %s\n' % d['t'])
def create_udp_listener(self):
if self.activeport != None:
# Port specified in the command line
port = self.activeport
else:
port = self.listener_port_setting.value
rfd = create_udp_socket('', port, False, reuse = True)
if rfd == None:
warning('Can not listen to UDP broadcasts on port %d\n' % port)
return
info('Listening to UDP broadcasts on port %d\n' % port)
rfd.setblocking(False)
io_add_watch(rfd, IO_IN, self.udp_listener_read)
def got_hello(self, d, address):
""" Check validity of Proximate hello, and register the other party. """
validator = {
'v': valid_protocol_version,
'pv': lambda x: type(x) == int and x >= 0,
'port': valid_port,
'nick': valid_nick,
'uid': lambda s: valid_uid(s) and s != self.myuid,
}
if not validate(validator, d):
if type(d) != dict or d.get('uid') != self.myuid:
info('Rejecting signature: %s\n' % str(d))
return
updatelist = [('nick', d['nick']), ('protocolversion', d['v'])]
if address != None:
ip = address[0]
else:
ip = None
self.add_or_update_user(d['uid'], updatelist, d['pv'], ip, d['port'])
def train_rnn(path_root,
start_tlen,
start_slen,
start_klen,
Tlen_max,
Slen_max,
Klen_max,
model_name='LSTM',
**kwargs):
import pickle
rnn_model = eval(model_name)
niters = 0
curr_klen = start_klen
curr_slen = start_slen
curr_tlen = start_tlen
def incr_tlen(l):
incr_l = min(8, l / 5) # cap large steps to 10
incr_l = max(incr_l, 1) # step at least 1
nxt_l = int(incr_l) + l
print(f'Increment Tlen. old_length is {l}. Next length is {nxt_l}')
if nxt_l <= Tlen_max:
return nxt_l
else:
return Tlen_max + 1
def incr_slen(l):
incr_l = min(3, l / 3) # cap large steps to 3
incr_l = max(incr_l, 1) #avoid fraction
nxt_l = int(incr_l) + l
print(f'Increment Slen. old_length is {l}. Next length is {nxt_l}')
if nxt_l <= Slen_max:
return nxt_l
else:
return Slen_max + 1
def incr_klen(l):
nxt_l = round(l + l / 2)
print(f'Increment klen. old_length is {l}. Next length is {nxt_l}')
if nxt_l <= Klen_max:
return nxt_l
else:
return Klen_max + 1
start_lr = kwargs['learning_rate']
# not sure stepping through K is a good idea.
# while curr_slen <= Slen_max:
iteration = 0
logs = {}
curr_klen = start_klen
while curr_slen <= Slen_max:
if curr_tlen > Tlen_max:
curr_tlen = Tlen_max
while curr_tlen <= Tlen_max:
info(' Current Klen = ' + str(curr_klen) + ' Slen = ' +
str(curr_slen) + ' Tlen = ' + str(curr_tlen))
hist[model_name] = copy_task(rnn_model,
model_name=model_name,
Tlen=curr_tlen,
Slen=curr_slen,
Klen=curr_klen,
**kwargs)
# save logs
logs[iteration] = [
hist[model_name].history['loss'], curr_tlen, curr_slen,
curr_klen
]
pickle_out = open(f"{path_root}{model_name}", "wb")
pickle.dump(logs, pickle_out)
iteration += 1
if hist[model_name].history['categorical_accuracy'][-1] < 0.96:
debug('Accuracy not high enough yet.')
debug('Repeat Klen = ' + str(curr_klen) + ' Slen = ' +
str(curr_slen) + ' Tlen = ' + str(curr_tlen))
if kwargs['learning_rate'] > 1e-6:
kwargs['learning_rate'] = kwargs['learning_rate'] / 2
debug('Lower max learning rate to ',
kwargs['learning_rate'])
else:
debug('Convergence failed. Ending')
return
niters = niters + 1
debug('Number of iterations = ', niters)
else:
niters = 0
printer('Done! Save specs and increase complexity.')
curr_tlen = incr_tlen(curr_tlen)
kwargs['learning_rate'] = start_lr
with open(best_specs_path, 'wb') as handle:
info('Dumping next iteration specs.')
pickle.dump((curr_klen, curr_slen, curr_tlen),
handle,
protocol=pickle.HIGHEST_PROTOCOL)
curr_slen = incr_slen(curr_slen)
printer('Klen = ' + str(curr_klen) + ' Slen = ' + str(curr_slen) +
' Tlen = ' + str(curr_tlen))
if (curr_slen == Slen_max) and (curr_klen
== Klen_max) and (curr_tlen
== Tlen_max):
printer('FINISHED!!!!')
printer('Klen = ' + str(curr_klen) + ' Slen = ' + str(curr_slen) +
' Tlen = ' + str(curr_tlen))
return
def add_task(RNN_model,
model_name='RNN',
activation='tanh',
length=400,
nb_layers=1,
nb_hid=128,
initializer_func=None,
hybrid_nunits=0,
hybrid_type=LSTM,
recurrent_initializer='orthogonal',
load_model=False,
N_train=1,
N_test=1,
model_path=None,
max_entries=3):
"""Perform the adding task
Parameters
----------
RNN_model : handle to the class
model_name: Just a name
length : int
nb_layers : int
number of recurrent layers
nb_hid : int
number of hidden units per layer
nb_epoch : int
total number of epoch
batch_size : int
the batch size
learning_rate : float
learning rate of RMSprop optimizer
clipnorm : float
gradient clipping, if >0
"""
# model_path = os.path.join('./results/masked_addition/',
# model_name + '_' + str(nb_hid) + '_' + str(nb_layers) + '.h5')
model_pic = os.path.join(model_path, model_name + "-model-pic.png")
# ----- print mode info -----
info("Model Name: ", model_name)
info("Number of layers: ", nb_layers)
info("Number of hidden units: ", nb_hid)
info("Activation: ", activation)
info("Recurrent initializer: ", recurrent_initializer)
# ----- prepare data -----
# identify data format
if K.backend() == "tensorflow":
K.set_image_data_format("channels_last")
else:
K.set_image_data_format("channels_first")
data_format = K.image_data_format()
X_train, Y_train, _, _ = load_adding_problem(length=length,
N_train=N_train,
N_test=N_test,
max_entries=max_entries,
save=False,
load=False)
info("Basic dataset statistics")
info("X_train shape:", X_train.shape)
info("Y_train shape:", Y_train.shape)
# setup sequence shape
input_shape = X_train.shape[1:]
# ----- Build Model -----
img_input = Input(shape=input_shape)
if initializer_func == None:
initializer_func = keras.initializers.Identity(gain=1.0)
x = define_model(RNN_model=RNN_model,
hybrid_nunits=hybrid_nunits,
h_dim=nb_hid,
op_dim=1,
num_layers=nb_layers,
activation=activation,
op_activation='linear',
recurrent_initializer=recurrent_initializer,
ip=img_input,
GRNN=hybrid_type,
op_type='sample',
learn_retention_ratio=True)
# compile model
info('Compiling model...')
model = Model(img_input, x)
model.summary()
if not os.path.isfile(model_path):
debug('ALERT - Model file does not exist. Ending.', model_path)
return
else:
model.load_weights(model_path, by_name=True)
# ---- Record activations -----
info(
'---------------------- Collecting the activations -------------------'
)
layer_outputs = [layer.output
for layer in model.layers[1:]] # 0 is the input layer
activation_model = Model(img_input, layer_outputs)
activations = activation_model.predict(X_train)
# printing weights of the network
names = [weight.name for layer in model.layers for weight in layer.weights]
weights = model.get_weights()
idx = 0
for name, weight in zip(names, weights):
print(name, weight.shape, idx)
idx += 1
idx = 0
if (RNN_model == LSTM):
print('ALERT: Makes assumptions about format. This may break!')
kernel = weights[0]
recurrent_kernel = weights[1]
bias = weights[2]
dense_kernel = weights[3]
dense_bias = weights[4]
# ------- Plotting gates ------
units = nb_hid
kernel_i = kernel[:, :units]
kernel_f = kernel[:, units:units * 2]
kernel_c = kernel[:, units * 2:units * 3]
kernel_o = kernel[:, units * 3:]
recurrent_kernel_i = recurrent_kernel[:, :units]
recurrent_kernel_f = recurrent_kernel[:, units:units * 2]
recurrent_kernel_c = recurrent_kernel[:, units * 2:units * 3]
recurrent_kernel_o = recurrent_kernel[:, units * 3:]
bias_i = bias[:units]
bias_f = bias[units:units * 2]
bias_c = bias[units * 2:units * 3]
bias_o = bias[units * 3:]
activation = eval(activation)
ractivation = hard_sigmoid
h_tm1 = np.zeros((1, units), dtype='float32')
c_tm1 = np.zeros((1, units), dtype='float32')
op = np.zeros(length, dtype='float32')
X_train = X_train.astype('float32')
fg = np.zeros((units, length))
ig = np.zeros((units, length))
og = np.zeros((units, length))
c_log = np.zeros((units, length))
h_log = np.zeros((units, length))
for idx in range(length):
inputs = X_train[:, idx, :]
x_i = np.dot(inputs, kernel_i)
x_f = np.dot(inputs, kernel_f)
x_c = np.dot(inputs, kernel_c)
x_o = np.dot(inputs, kernel_o)
x_i = np.add(x_i, bias_i)
x_f = np.add(x_f, bias_f)
x_c = np.add(x_c, bias_c)
x_o = np.add(x_o, bias_o)
h_tm1_i = h_tm1
h_tm1_f = h_tm1
h_tm1_c = h_tm1
h_tm1_o = h_tm1
i = ractivation(np.add(x_i, np.dot(h_tm1_i, recurrent_kernel_i)))
f = ractivation(np.add(x_f, np.dot(h_tm1_f, recurrent_kernel_f)))
c = np.add(
f * c_tm1,
i *
activation(np.add(x_c, np.dot(h_tm1_c, recurrent_kernel_c))))
o = ractivation(np.add(x_o, np.dot(h_tm1_o, recurrent_kernel_o)))
h = o * activation(c)
h_tm1 = h + 0
c_tm1 = c + 0
fg[:,
idx], ig[:,
idx], og[:,
idx], c_log[:,
idx], h_log[:,
idx] = f, i, o, c, h
idx = 1
# for idx in range(units):
plt.plot(fg[idx, :], label='Forget gate')
plt.plot(ig[idx, :], label='Input gate')
plt.plot(og[idx, :], label='Output gate')
plt.plot(c_log[idx, :], label='Internal state')
plt.plot(h_log[idx, :], label='Output')
legend_properties = {'weight': 'bold'}
plt.plot(X_train[0, :, 1], '*', label='Mask')
plt.legend(prop=legend_properties)
plt.savefig('LSTM_mask_add.pdf', dpi=500)
# Dense layer computation
op = np.dot(h_tm1, dense_kernel)
op = np.add(op, dense_bias)
if (RNN_model == lpRNN):
op = np.zeros(length, dtype='float32')
rnn_activation = activations[0]
dense_kernel = weights[4]
dense_bias = weights[5]
for idx in range(length):
op[idx] = np.dot(rnn_activation[0, idx, :], dense_kernel)
op[idx] += dense_bias
plt.plot(X_train[0, :, 1], '*', label='Mask')
print(op.shape)
for idx in range(nb_hid):
plt.plot(rnn_activation[0, :, idx], alpha=0.6, lw=1)
plt.plot(op, label='Activation', lw=3)
plt.xlabel(r'Time step', weight='bold')
plt.ylabel(r'Value', weight='bold')
plt.show()
# ----- Inference run -----
output = model.predict(X_train)
info(f'Expected output = {Y_train}. Actual output = {output}')
return output
def add_task(RNN_model,
model_name='RNN',
activation='tanh',
length=400,
nb_layers=1,
nb_hid=128,
nb_epoch=10,
batch_size=32,
learning_rate=0.01,
clipnorm=1000,
initializer_func=None,
hybrid_nunits=0,
hybrid_type=LSTM,
recurrent_initializer='orthogonal',
gate_regularizer=None,
learn_retention_ratio=False,
load_model=False,
N_train=20000,
N_test=1000,
model_path=None,
max_entries=3):
"""Perform the adding task
Parameters
----------
RNN_model : handle to the class
model_name: Just a name
length : int
nb_layers : int
number of recurrent layers
nb_hid : int
number of hidden units per layer
nb_epoch : int
total number of epoch
batch_size : int
the batch size
learning_rate : float
learning rate of RMSprop optimizer
clipnorm : float
gradient clipping, if >0
"""
# model_path = os.path.join('./results/masked_addition/',
# model_name + '_' + str(nb_hid) + '_' + str(nb_layers) + '.h5')
model_pic = os.path.join(model_path, model_name + "-model-pic.png")
# ----- print mode info -----
info("Model Name: ", model_name)
info("Number of epochs: ", nb_epoch)
info("Batch Size: ", batch_size)
info("Number of layers: ", nb_layers)
info("Number of hidden units: ", nb_hid)
info("Activation: ", activation)
info("Recurrent initializer: ", recurrent_initializer)
# ----- prepare data -----
# identify data format
if K.backend() == "tensorflow":
K.set_image_data_format("channels_last")
else:
K.set_image_data_format("channels_first")
data_format = K.image_data_format()
X_train, Y_train, X_test, Y_test = load_adding_problem(
length=length, N_train=N_train, N_test=N_test, max_entries=max_entries)
info("Basic dataset statistics")
info("X_train shape:", X_train.shape)
info("Y_train shape:", Y_train.shape)
info("X_test shape:", X_test.shape)
info('Y_test shape:', Y_test.shape)
# setup sequence shape
input_shape = X_train.shape[1:]
# ----- Build Model -----
img_input = Input(shape=input_shape)
if initializer_func == None:
initializer_func = keras.initializers.Identity(gain=1.0)
x = define_model(RNN_model=RNN_model,
hybrid_nunits=hybrid_nunits,
h_dim=nb_hid,
op_dim=1,
num_layers=nb_layers,
activation=activation,
op_activation='linear',
recurrent_initializer=recurrent_initializer,
ip=img_input,
GRNN=hybrid_type,
op_type='sample',
learn_retention_ratio=learn_retention_ratio,
gate_regularizer=gate_regularizer)
# compile model
info('Compiling model...')
model = Model(img_input, x)
model.summary()
if not os.path.isfile(model_path):
if load_model == True:
debug('File does not exist. Creating new.')
else:
model.load_weights(model_path, by_name=True)
# ----- Configure Optimizer -----
# opt = RMSprop(lr=learning_rate, clipnorm=clipnorm)
opt = SGD(lr=learning_rate, clipnorm=clipnorm)
model.compile(loss='mse', optimizer=opt, metrics=['mse'])
print("[MESSAGE] Model is compiled.")
# Callbacks
early_stop = EarlyStopping(monitor="val_loss", patience=25, verbose=1)
print_model_name = LambdaCallback(on_epoch_begin=lambda batch, logs: info(
'Running ' + model_name + ', Add task length = ' + str(length)))
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.9,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=1e-6)
stoponloss = StopOnLoss(monitor='loss', value=1e-3, verbose=1)
checkpoint = MyModelCheckpoint(model_path,
monitor='loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min')
# ----- Training Model -----
# Fit the model on the batches generated by datagen.flow().
history = model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
validation_data=(X_test, Y_test),
callbacks=[
reduce_lr, early_stop, print_model_name,
checkpoint, stoponloss
])
return history
def ok_dialog(self, headline, msg, destroy_cb=None, parent=None, modal=False):
info("OK dialog: %s: %s\n" % (headline, msg))
if self.ui != None:
self.ui.ok_dialog(headline, msg, destroy_cb, parent, modal)
def copy_task(RNN_model,
model_name='RNN',
activation='tanh',
nb_layers=1,
nb_hid=128,
nb_epoch=3,
batch_size=32,
learning_rate=0.1,
nbatches=100,
clipnorm=10,
momentum=0,
initializer_func=None,
hybrid_nunits=0,
hybrid_type=LSTM,
Tlen=100,
Slen=20,
Klen=128,
NClasses=1024,
recurrent_initializer='orthogonal',
learn_retention_ratio=False,
load_model=False,
model_path=None):
"""Perform LSTM The copying Problem experiment.
Parameters
----------
RNN_model : handle to the class
model_name: Just a name
nb_layers : int
number of recurrent layers
nb_hid : int
number of hidden units per layer
nb_epoch : int
total number of epoch
batch_size : int
the batch size
learning_rate : float
learning rate of RMSprop optimizer
clipnorm : float
gradient clipping, if >0
"""
# the model layout picture
model_pic = os.path.join(model_path, model_name + "-model-pic.png")
# ----- print mode info -----
# info("Model Name: ", model_name)
# info("Number of epochs: ", nb_epoch)
# info("Batch Size: ", batch_size)
# info("Number of layers: ", nb_layers)
# info("Number of hidden units: ", nb_hid)
# info("Activation: ", activation)
# info("Recurrent initializer: ", recurrent_initializer)
train_gen = data_generator(nsamples=batch_size,
Tlen=Tlen,
Slen=Slen,
Klen=Klen,
NClasses=NClasses)
val_gen = data_generator(nsamples=int(batch_size / 2),
Tlen=Tlen,
Slen=Slen,
Klen=Klen,
NClasses=NClasses)
X_train, Y_train, _ = next(train_gen)
X_test, Y_test, _ = next(val_gen)
# info("Basic dataset statistics")
# info("X_train shape:", X_train.shape)
# info("Y_train shape:", Y_train.shape)
# info("X_test shape:", X_test.shape)
# info('Y_test shape:', Y_test.shape)
# setup sequence shape
input_shape = X_train.shape[1:]
# ----- Build Model -----
img_input = Input(shape=input_shape)
if initializer_func == None:
initializer_func = keras.initializers.Identity(gain=1.0)
x = define_model(RNN_model=RNN_model,
hybrid_nunits=hybrid_nunits,
h_dim=nb_hid,
op_dim=NClasses,
num_layers=nb_layers,
activation=activation,
op_activation='softmax',
recurrent_initializer=recurrent_initializer,
ip=img_input,
GRNN=hybrid_type,
op_type='seq',
learn_retention_ratio=learn_retention_ratio)
# compile model
print("[MESSAGE] Compiling model...")
model = Model(img_input, x)
model.summary()
if not os.path.isfile(model_path):
if load_model == True:
debug('File does not exist. Creating new.')
else:
model.load_weights(model_path, by_name=True)
# ----- Configure Optimizer -----
# rmsprop = RMSprop(lr=learning_rate, clipnorm=clipnorm)
opt = SGD(lr=learning_rate, clipnorm=clipnorm, momentum=momentum)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=[keras.metrics.categorical_accuracy],
sample_weight_mode="temporal")
print("[MESSAGE] Model is compiled.")
# Callbacks
early_stop = EarlyStopping(
monitor="loss",
patience=10,
verbose=1,
min_delta=0.0001,
)
print_model_name = LambdaCallback(on_epoch_begin=lambda batch, logs: info(
'Running ' + model_name + ', Copy task Slen = ' + str(
Slen) + ', Klen = ' + str(Klen) + ', Tlen = ' + str(Tlen)))
reduce_lr = ReduceLROnPlateau(monitor='loss',
factor=0.5,
patience=5,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=1e-7)
checkpoint = MyModelCheckpoint(model_path,
monitor='loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='min')
stoponacc = StopOnAcc(monitor='categorical_accuracy',
value=0.99,
verbose=1)
stoponloss = StopOnLoss(monitor='loss', value=1e-5, verbose=1)
# ----- Training Model -----
history = model.fit_generator(generator=train_gen,
validation_data=val_gen,
epochs=nb_epoch,
steps_per_epoch=nbatches,
validation_steps=1,
callbacks=[
reduce_lr, early_stop, print_model_name,
checkpoint, stoponacc, stoponloss
])
# Testing just for visualization
test_gen = data_generator(nsamples=1,
Tlen=Tlen,
Slen=Slen,
Klen=Klen,
NClasses=NClasses)
X_test, Y_test, _ = next(test_gen)
pred_ = model.predict(X_test)
prediction = np.argmax(pred_, axis=-1)
tgt = np.argmax(Y_test, axis=-1)
# Plotting the results
# print('Plotting for case ' + model_name)
# plt.figure()
# plt.plot(prediction[0, :], 'r', alpha=0.6, label=model_name + 'Prediction')
# plt.legend()
# plt.plot(tgt[0, :], 'k', label='Target')
# plt.legend()
# plt.show()
return history
'hybrid_type': args.hybrid_type,
'load_model': args.load_model,
'learning_rate': args.lr,
'clipnorm': args.clipnorm,
'momentum': args.momentum,
'nb_epoch': args.nb_epoch,
'batch_size': args.batch_size,
'nbatches': args.nbatches,
'NClasses': args.nclasses,
'activation': 'relu'
}
print("Number of epochs is ", kwargs[rnn_model_name]['nb_epoch'])
# Load best specs on record.
if load_specs and os.path.isfile(best_specs_path):
info('Loading previous run specs')
with open(best_specs_path, 'rb') as handle:
start_klen, start_slen, start_tlen = pickle.load(handle)
info('Start Klen = ' + str(start_klen) + ', Slen = ' +
str(start_slen) + ',Tlen = ' + str(start_tlen))
else:
start_tlen = args.Tstart
start_slen = args.Sstart
start_klen = args.Kstart
if start_klen > Kmax:
start_klen = Kmax
if start_tlen > Tmax:
start_tlen = Tmax
if start_slen > Smax:
start_slen = Smax
