def train(cfg): model = IBnet(cfg) dataset = IBDataset(cfg['NAME']) dataloader = DataLoader(dataset, batch_size=cfg['SGD_BATCHSIZE'], shuffle=False, num_workers=0) optimizer = torch.optim.Adam(model.parameters(), lr=cfg['SGD_LEARNINGRATE']) Loss = torch.nn.CrossEntropyLoss() full = construct_full_dataset(dataset.trn, dataset.tst) for epoch in range(cfg['NUM_EPOCHS']): for batch_id, (data, label) in enumerate(dataloader): output = model(data) loss = Loss(output, label) optimizer.zero_grad() loss.backward() optimizer.step() if batch_id == 0: # every epoch report once max_element = model.report(full.X, epoch) if max_element > cfg['MAX_ELEMENT']: cfg['MAX_ELEMENT'] = max_element if epoch in cfg['EPOCHS']: test_accuracy = model.get_accuracy(dataset.tst.X, dataset.tst.y) print('epoch:{}, loss:{}, test_accuracy:{}'.format( epoch, loss, test_accuracy))
def __init__(self, train, test, log_dir='./logs', embeddings_freq=10): """ Args: train: The training data test: The test data log_dir: Path to directory used for logging embeddings_freq: Defines how often embedding variables will be saved to the log directory. If set to 1, this is done every epoch, if it is set to 10 every 10th epoch and so forth. """ super().__init__() self.sess = K.get_session() self.log_dir = log_dir self.embeddings_freq = embeddings_freq self.writer = tf_summary.FileWriter(self.log_dir) self.saver: tf.train.Saver self.embeddings_ckpt_path = os.path.join(self.log_dir, 'keras_embedding.ckpt') self.train = train self.test = test self.full = utils.construct_full_dataset(train, test) # Save metadata. np.savetxt(f'{log_dir}/metadata.tsv', self.test.y, fmt='%i')
def __init__(self, cfg, trn, tst, do_save_func=None, *kargs, **kwargs): super(LoggingReporter, self).__init__(*kargs, **kwargs) self.cfg = cfg # Configuration options dictionary self.trn = trn # Train data self.tst = tst # Test data if 'FULL_MI' not in cfg: self.cfg['FULL_MI'] = False # Whether to compute MI on train and test data, or just test if self.cfg['FULL_MI']: self.full = utils.construct_full_dataset(trn,tst) # do_save_func(epoch) should return True if we should save on that epoch self.do_save_func = do_save_func
entropy_func_lower = K.function([ Klayer_activity, ], [ kde.entropy_estimator_bd(Klayer_activity, noise_variance), ]) # nats to bits conversion factor nats2bits = 1.0 / np.log(2) # Save indexes of tests data for each of the output classes saved_labelixs = {} y = tst.y Y = tst.Y if FULL_MI: full = utils.construct_full_dataset(trn, tst) y = full.y Y = full.Y for i in range(NUM_LABELS): saved_labelixs[i] = y == i labelprobs = np.mean(Y, axis=0) # In[14]: PLOT_LAYERS = None # Which layers to plot. If None, all saved layers are plotted # Data structure used to store results measures = OrderedDict() measures['tanh'] = {}
def computeMI(id): train, test = utils.get_IB_data('2017_12_21_16_51_3_275766') # For both train and test musr correspond with saving code FULL_MI = True # MI Measure infoplane_measure = 'bin' DO_SAVE = True DO_LOWER = (infoplane_measure == 'lower') DO_BINNED = (infoplane_measure == 'bin') MAX_EPOCHS = 5000 NUM_LABELS = 2 COLORBAR_MAX_EPOCHS = 5000 # Directory For Loading saved Data ARCH = '10-7-5-4-3' DIR_TEMPLATE = '%%s_%s' % ARCH noise_variance = 1e-3 binsize = 0.07 Klayer_activity = K.placeholder(ndim=2) entropy_func_upper = K.function([ Klayer_activity, ], [ kde.entropy_estimator_kl(Klayer_activity, noise_variance), ]) entropy_func_lower = K.function([ Klayer_activity, ], [ kde.entropy_estimator_bd(Klayer_activity, noise_variance), ]) # Nats to bits conversion nats2bits = 1.0 / np.log(2) # Indexes of tests data for each of Output Classes saved_labelixs = {} y = test.y Y = test.Y if FULL_MI: full = utils.construct_full_dataset(train, test) y = full.y Y = full.Y for i in range(NUM_LABELS): saved_labelixs[i] = (y == i) labelprobs = np.mean(Y, axis=0) # Layers to plot, None for all PLOT_LAYERS = None # Store Results measures = OrderedDict() measures['tanh'] = {} measures['relu'] = {} for activation in measures.keys(): cur_dir = 'rawdata/' + DIR_TEMPLATE % activation if not os.path.exists(cur_dir): print("Directory %s not found" % cur_dir) continue print("******* Loading %s ******" % cur_dir) for epochfile in sorted(os.listdir(cur_dir)): if not epochfile.startswith('epoch'): continue fname = cur_dir + '/' + epochfile with open(fname, 'rb') as f: d = cPickle.load(f) epoch = d['epoch'] if epoch in measures[activation]: continue if epoch > MAX_EPOCHS: continue print("Measureing ", fname) num_layers = len(d['data']['activity_tst']) if PLOT_LAYERS is None: PLOT_LAYERS = [] for lndx in range(num_layers): PLOT_LAYERS.append(lndx) cepochdata = defaultdict(list) for lndx in range(num_layers): activity = d['data']['activity_tst'][lndx] h_upper = entropy_func_upper([ activity, ])[0] if DO_LOWER: h_lower = entropy_func_lower() hM_given_X = kde.kde_condentropy(activity, noise_variance) hM_given_Y_upper = 0 for i in range(NUM_LABELS): hcond_upper = entropy_func_upper([ activity[saved_labelixs[i], :], ])[0] hM_given_Y_upper += labelprobs[i] * hcond_upper if DO_LOWER: hM_given_Y_lower = 0 for i in range(NUM_LABELS): hcond_lower = entropy_func_lower([ activity[saved_labelixs[i], :], ])[0] hM_given_Y_lower += labelprobs[i] * hcond_lower cepochdata['MI_XM_upper'].append(nats2bits * (h_upper - hM_given_X)) cepochdata['MI_YM_upper'].append(nats2bits * (h_upper - hM_given_Y_upper)) cepochdata['H_M_upper'].append(nats2bits * h_upper) pstr = 'upper: MI(X;M)=%0.3f, MI(Y;M)=%0.3f' % ( cepochdata['MI_XM_upper'][-1], cepochdata['MI_YM_upper'][-1]) if DO_LOWER: cepochdata['MI_XM_lower'].append(nats2bits * (h_lower - hM_given_X)) cepochdata['MI_YM_lower'].append( nats2bits * (h_lower - hM_given_Y_lower)) cepochdata['H_M_lower'].append(nats2bits * h_lower) pstr += 'lower: MI(X;M)=%0.3f, MI(Y;M)=%0.3f' % ( cepochdata['MI_XM_lower'][-1], cepochdata['MI_YM_lower'][-1]) if DO_BINNED: binxm, binym = simplebinmi.bin_calc_information2( saved_labelixs, activity, binsize) cepochdata['MI_XM_bin'].append(nats2bits * binxm) cepochdata['MI_YM_bin'].append(nats2bits * binym) pstr += 'bin: MI(X;M)=%0.3f, MI(Y;M)=%0.3f' % ( cepochdata['MI_XM_bin'][-1], cepochdata['MI_YM_bin'][-1]) print('- Layer %d %s' % (lndx, pstr)) measures[activation][epoch] = cepochdata with open("MI" + str(id), 'wb') as f: cPickle.dump(measures, f)
output = model(data) loss = Loss(output, label) optimizer.zero_grad() loss.backward() optimizer.step() if batch_id == 0: # every epoch report once max_element = model.report(full.X, epoch) if max_element > cfg['MAX_ELEMENT']: cfg['MAX_ELEMENT'] = max_element if epoch in cfg['EPOCHS']: test_accuracy = model.get_accuracy(dataset.tst.X, dataset.tst.y) print('epoch:{}, loss:{}, test_accuracy:{}'.format( epoch, loss, test_accuracy)) if __name__ == "__main__": cfg = get_cfg() #train(cfg) print('--------max_elements:{}---------------'.format(cfg['MAX_ELEMENT'])) dataset = IBDataset(cfg['NAME']) full = construct_full_dataset(dataset.trn, dataset.tst) measures = MI(cfg, infoplane_measure='HSIC', full_dataset=full) plot_infoplane(cfg, infoplane_measure='HSIC', measures=measures, PLOT_LAYERS=[0, 1, 2, 3, 4]) #plot_SNR(cfg, PLOT_LAYERS=[0,1,2,3,4])