def back_prop(self, cm_act_grad, cm_data, cm_input_grad=None):
'''
back prop activation grad, and compute gradients.
'''
cm.dot(cm_data, cm_act_grad.T, self._wts_grad)
if cm_input_grad is not None:
cm.dot(self._wts, cm_act_grad, cm_input_grad)
cm_act_grad.sum(axis=1, target=self._b_grad)
self._wts_grad.divide(cm_data.shape[1])
self._b_grad.divide(cm_data.shape[1])
def compute_recon_error_for_db(self, data_src, batch_size):
err_sum, num_pts = 0, 0
for batch_data in data_src.get_iterator(batch_size, return_labels=False):
######### START POSITIVE PHASE ############
try:
cm_hidprobs
except NameError:
cm_hidprobs = cm.empty((self.num_units, batch_size))
cm_recon = cm.empty(batch_data.shape).assign(0)
cm.dot(self.cmW.T, batch_data, cm_hidprobs)
cm_hidprobs.add_col_vec(self.cmBiasesHid)
cm_hidprobs.apply_sigmoid()
cm.dot(self.cmW, cm_hidprobs, target=cm_recon)
cm_recon.add_col_vec(self.cmBiasesVis)
cm_recon.subtract(batch_data)
err = cm_recon.euclid_norm()**2
err_sum = err + err_sum
num_pts = num_pts + batch_size
return sqrt(err_sum*1./(self.input_dim*num_pts))
def project(self, cm_data, cm_hidprobs):
cm.dot(self.cmW.T, cm_data, cm_hidprobs)
cm_hidprobs.add_col_vec(self.cmBiasesHid)
cm_hidprobs.apply_sigmoid()
def train_cd1_for_epoch(self, data_src, dev_src, batch_size, momentum,
epsilonw, epsilonvb, epsilonhb, l2_decay):
batch, err_sum, total_active, num_batch_cnts = 0, 0, 0, 0
cm_recon, cm_hidprobs, cm_hidstates, cm_posprods, cm_negprods, \
cm_poshidacts, cm_neghidacts, cm_posvisacts, cm_negvisacts = \
self.allocate_activations(batch_size)
printStr, printStrNew = "",""
for batch_data in data_src.get_iterator(batch_size, return_labels=False):
batch = batch + 1
######### START POSITIVE PHASE ############
cm.dot(self.cmW.T, batch_data, cm_hidprobs)
cm_hidprobs.add_col_vec(self.cmBiasesHid)
cm_hidprobs.apply_sigmoid()
cm_hidstates.fill_with_rand()
cm_hidstates.less_than(cm_hidprobs)
num_active = cm_hidstates.euclid_norm()**2
if batch % 20 == 0:
cm.dot(self.cmW, cm_hidprobs, target=cm_recon)
cm_recon.add_col_vec(self.cmBiasesVis)
cm_recon.subtract(batch_data)
err = cm_recon.euclid_norm()**2
err_sum = err + err_sum
num_batch_cnts = num_batch_cnts + 1
avg_err = sqrt(err_sum/(num_batch_cnts*batch_size*self.input_dim))
printStr = "Epoch = %d, batch = %d, Avg Err = %.3f"%(\
self.epoch, batch, avg_err)
printString = printStrNew + printStr
sys.stderr.write(printString)
sys.stderr.flush()
printStrNew = '\b' * (len(printStr))
cm.dot(batch_data, cm_hidprobs.T, target=cm_posprods)
cm_hidprobs.sum(axis=1, target=cm_poshidacts)
batch_data.sum(axis=1, target=cm_posvisacts)
######### START NEGATIVE PHASE#########
cm.dot(self.cmW, cm_hidstates, target=cm_recon)
cm_recon.add_col_vec(self.cmBiasesVis)
cm.dot(self.cmW.T, cm_recon, target=cm_hidprobs)
cm_hidprobs.add_col_vec(self.cmBiasesHid)
cm_hidprobs.apply_sigmoid()
cm.dot(cm_recon, cm_hidprobs.T, target=cm_negprods)
cm_hidprobs.sum(axis=1, target=cm_neghidacts)
cm_recon.sum(axis=1, target=cm_negvisacts)
self.cmWInc.mult(momentum)
cm_posprods.subtract(cm_negprods)
self.cmWInc.add_mult(cm_posprods, epsilonw/batch_size)
cm_poshidacts.subtract(cm_neghidacts)
self.cmBiasesHidInc.mult(momentum)
self.cmBiasesHidInc.add_mult(cm_poshidacts, epsilonhb/batch_size)
cm_posvisacts.subtract(cm_negvisacts)
self.cmBiasesVisInc.mult(momentum)
self.cmBiasesVisInc.add_mult(cm_posvisacts, epsilonvb/batch_size)
self.cmW.add_mult(self.cmW, -l2_decay*epsilonw)
self.cmW.add(self.cmWInc)
self.cmBiasesHid.add(self.cmBiasesHidInc)
self.cmBiasesVis.add(self.cmBiasesVisInc)
total_active = total_active + num_active
avg_active = total_active/(batch*batch_size*self.num_units)
avg_err = sqrt(err_sum/(num_batch_cnts*batch_size*self.input_dim))
logging.info("Epoch # %d, batch %d, avg(err) = %.4f, avg(on) = %.3f"%(\
self.epoch, batch, avg_err, avg_active))
sys.stderr.write("\nEpoch # %d, batch %d, avg(err) = %.4f, avg(on) = %.3f\n"%(\
self.epoch, batch, avg_err, avg_active))
dev_error = self.compute_recon_error_for_db(dev_src, batch_size)
logging.info("Epoch # %d, DEV SET avg(err) = %.4f"%(self.epoch,dev_error))
sys.stderr.write("Epoch # %d, DEV SET avg(err) = %.4f\n"%(self.epoch,dev_error))
return dev_error
def project(self, cm_data, cm_act):
cm.dot(self._wts.T, cm_data, cm_act)
if self.dropout != 0:
cm_act.mult(1. / (1. - self.dropout))
cm_act.add_col_vec(self._b)