def optimize(self, inputs, extra_inputs=None, callback=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(inputs,
self._batch_size,
extra_inputs=extra_inputs)
sess = tf.get_default_session()
for epoch in range(self._max_epochs):
if self._verbose:
logger.log("Epoch %d" % (epoch))
progbar = pyprind.ProgBar(len(inputs[0]))
gc.collect()
for batch in dataset.iterate(update=True):
sess.run(self._train_op,
dict(list(zip(self._input_vars, batch))))
if self._verbose:
progbar.update(len(batch[0]))
gc.collect()
if self._verbose:
if progbar.active:
progbar.stop()
new_loss = f_loss(*(tuple(inputs) + extra_inputs))
if self._verbose:
logger.log("Epoch: %d | Loss: %f" % (epoch, new_loss))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(
trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize(self, inputs, extra_inputs=None, callback=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(inputs, self._batch_size, extra_inputs=extra_inputs)
sess = tf.get_default_session()
for epoch in range(self._max_epochs):
if self._verbose:
logger.log("Epoch %d" % (epoch))
progbar = pyprind.ProgBar(len(inputs[0]))
for batch in dataset.iterate(update=True):
if self._init_train_op is not None:
sess.run(self._init_train_op, dict(list(zip(self._input_vars, batch))))
self._init_train_op = None # only use it once
else:
sess.run(self._train_op, dict(list(zip(self._input_vars, batch))))
if self._verbose:
progbar.update(len(batch[0]))
if self._verbose:
if progbar.active:
progbar.stop()
new_loss = f_loss(*(tuple(inputs) + extra_inputs))
if self._verbose:
logger.log("Epoch: %d | Loss: %f" % (epoch, new_loss))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize_gen(self,
inputs,
extra_inputs=None,
callback=None,
yield_itr=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_opt = self._opt_fun["f_opt"]
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(inputs,
self._batch_size,
extra_inputs=extra_inputs
#, randomized=self._randomized
)
itr = 0
for epoch in pyprind.prog_bar(list(range(self._max_epochs))):
for batch in dataset.iterate(update=True):
f_opt(*batch)
if yield_itr is not None and (itr % (yield_itr + 1)) == 0:
yield
itr += 1
new_loss = self.loss(inputs, extra_inputs)
if self._verbose:
logger.log("Epoch %d, loss %s" % (epoch, new_loss))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(
trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize_gen(self, inputs, extra_inputs=None, callback=None, yield_itr=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_opt = self._opt_fun["f_opt"]
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(
inputs, self._batch_size,
extra_inputs=extra_inputs
#, randomized=self._randomized
)
itr = 0
for epoch in pyprind.prog_bar(list(range(self._max_epochs))):
for batch in dataset.iterate(update=True):
f_opt(*batch)
if yield_itr is not None and (itr % (yield_itr+1)) == 0:
yield
itr += 1
new_loss = f_loss(*(tuple(inputs) + extra_inputs))
if self._verbose:
logger.log("Epoch %d, loss %s" % (epoch, new_loss))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize(self, inputs, extra_inputs=None, callback=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_opt = self._opt_fun["f_opt"]
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(inputs,
self._batch_size,
extra_inputs=extra_inputs)
for epoch in xrange(self._max_epochs):
if self._verbose:
logger.log("Epoch %d" % epoch)
for batch in dataset.iterate(update=True):
f_opt(*batch)
new_loss = f_loss(*(tuple(inputs) + extra_inputs))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(
trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize(self, inputs, extra_inputs=None, callback=None):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
f_opt = self._opt_fun["f_opt"]
f_loss = self._opt_fun["f_loss"]
if extra_inputs is None:
extra_inputs = tuple()
last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
dataset = BatchDataset(inputs, self._batch_size, extra_inputs=extra_inputs)
for epoch in range(self._max_epochs):
if self._verbose:
logger.log("Epoch %d" % epoch)
for batch in dataset.iterate(update=True):
f_opt(*batch)
new_loss = f_loss(*(tuple(inputs) + extra_inputs))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=new_loss,
params=self._target.get_param_values(trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)
if abs(last_loss - new_loss) < self._tolerance:
break
last_loss = new_loss
def optimize(self,
inputs,
extra_inputs=None,
callback=None,
val_inputs=[None],
val_extra_inputs=tuple([None])):
if len(inputs) == 0:
# Assumes that we should always sample mini-batches
raise NotImplementedError
assert len(inputs) == 1
dataset_size, _ = inputs[0].shape
f_loss = self._opt_fun["f_loss"]
# Plot individual costs from complexity / likelihood terms.
try:
use_c_loss = True
c_loss = self._opt_fun["c_loss"]
except KeyError:
use_c_loss = False
try:
use_l_loss = True
l_loss = self._opt_fun["l_loss"]
except KeyError:
use_l_loss = False
if extra_inputs is None:
extra_inputs = tuple()
#last_loss = f_loss(*(tuple(inputs) + extra_inputs))
start_time = time.time()
train_dataset = BatchDataset(inputs,
self._batch_size,
extra_inputs=extra_inputs)
if not all([vi is None for vi in val_inputs]):
val_dataset = BatchDataset(val_inputs,
self._batch_size,
extra_inputs=val_extra_inputs)
sess = tf.get_default_session()
for epoch in range(self._max_epochs):
if self._verbose:
logger.log("Epoch %d" % (epoch))
progbar = pyprind.ProgBar(len(inputs[0]))
train_losses = []
train_c_losses, train_l_losses = [], []
# batch is a (matrix X, matrix Y) tuple
for t, batch in enumerate(train_dataset.iterate(update=True)):
sess.run(self._train_op,
dict(list(zip(self._input_vars, batch))))
train_losses.append(f_loss(*batch))
train_c_losses.append(c_loss(*batch))
train_l_losses.append(l_loss(*batch))
if self._verbose:
progbar.update(len(batch[0]))
train_loss = np.mean(train_losses)
train_c_loss = np.mean(train_c_losses)
train_l_loss = np.mean(train_l_losses)
val_losses = []
if not all([vi is None for vi in val_inputs]):
for t, batch in enumerate(val_dataset.iterate(update=True)):
val_losses.append(f_loss(*batch))
val_loss = np.mean(val_losses)
for interval, op in self._update_priors_ops:
if interval != 0 and epoch % interval == 0:
sess.run(op)
if self._verbose:
if progbar.active:
progbar.stop()
if self._verbose:
logger.log("Epoch: %d | Loss: %f" % (epoch, train_loss))
if self._callback or callback:
elapsed = time.time() - start_time
callback_args = dict(
loss=train_loss,
params=self._target.get_param_values(
trainable=True) if self._target else None,
itr=epoch,
elapsed=elapsed,
)
if use_c_loss:
callback_args['c_loss'] = train_c_loss
if use_l_loss:
callback_args['l_loss'] = train_l_loss
if val_loss is not None:
callback_args.update({'val_loss': val_loss})
if self._callback:
self._callback(callback_args)
if callback:
callback(**callback_args)