def __init__(self): super().__init__() layer0 = BernoulliLayer( torch.nn.Parameter(torch.zeros([1, 1, 28, 28]), requires_grad=True)) layer1 = BernoulliLayer( torch.nn.Parameter(torch.zeros([1, 200]), requires_grad=True)) layer2 = BernoulliLayer( torch.nn.Parameter(torch.zeros([1, 200]), requires_grad=True)) interactionUp1 = InteractionLinear(layer0.bias.shape[1:], layer1.bias.shape[1:]) interactionDown1 = InteractionLinear(layer1.bias.shape[1:], layer0.bias.shape[1:]) interactionUp2 = InteractionLinear(layer1.bias.shape[1:], layer2.bias.shape[1:]) interactionDown2 = InteractionLinear(layer2.bias.shape[1:], layer1.bias.shape[1:]) parameters = chain(*[ m.parameters() for m in [ layer0, layer1, layer2, interactionUp1, interactionUp2, interactionDown1, interactionDown2 ] ]) opt = torch.optim.Adam(parameters) self.model = HelmholtzMachine([layer0, layer1, layer2], [interactionUp1, interactionUp2], [interactionDown1, interactionDown2], optimizer=opt)
def __init__(self): super().__init__() layer0 = BernoulliLayer(torch.zeros([1, 1, 28, 28], requires_grad=True)) layer1 = BernoulliLayer( torch.zeros([1, 40, 17, 17], requires_grad=True)) interaction0 = InteractionModule(torch.nn.Conv2d(1, 40, 12)) rbm1 = RestrictedBoltzmannMachinePCD(layer0, layer1, interaction0, fantasy_particles=10) opt = torch.optim.Adam(rbm1.parameters(), lr=1e-3) self.model = DeepBeliefNetwork([rbm1], opt)
def __init__(self): super().__init__() layer0 = BernoulliLayer(torch.zeros([1, 1, 28, 28], requires_grad=True)) layer1 = CategoricalLayer( torch.zeros([1, 40, 8, 8, 5], requires_grad=True)) layer2 = CategoricalLayer( torch.zeros([1, 40, 1, 1, 5], requires_grad=True)) interaction0 = InteractionSequential( InteractionModule(torch.nn.Conv2d(1, 40, 12)), InteractionPoolMapIn2D(2, 2)) interaction1 = InteractionSequential( InteractionPoolMapOut2D(2, 2), InteractionModule(torch.nn.Conv2d(40, 40, 6)), InteractionPoolMapIn2D(2, 2)) rbm1 = RestrictedBoltzmannMachinePCD(layer0, layer1, interaction0, fantasy_particles=10) rbm2 = RestrictedBoltzmannMachinePCD(layer1, layer2, interaction1, fantasy_particles=10) opt = torch.optim.Adam(chain(rbm1.parameters(), rbm2.parameters()), lr=1e-3) self.model = DeepBeliefNetwork([rbm1, rbm2], opt)
def __init__(self): ''' Builds the DBM with hidden dimension = 200, 200. ''' super().__init__() # initialize the bias with zeros and lock them. # This is because the DBM approximates the bias distribution with higher layers. l0bias = torch.zeros([1, 1, 28, 28]) l0bias.requires_grad = False l1bias = torch.zeros([1, 200]) l1bias.requires_grad = False l2bias = torch.zeros([1, 200]) l2bias.requires_grad = False # initialize the bernoulli layers l0 = BernoulliLayer(l0bias) l1 = BernoulliLayer(l1bias) l2 = BernoulliLayer(l2bias) # initialize the interaction layers i0 = InteractionLinear(l0.bias.shape[1:], l1.bias.shape[1:]) i1 = InteractionLinear(l1.bias.shape[1:], l2.bias.shape[1:]) # build two RBMs rbm0 = RestrictedBoltzmannMachineCD_Smooth(l0, l1, i0, ksteps=1) rbm1 = RestrictedBoltzmannMachineCD_Smooth(l1, l2, i1, ksteps=1) # get all parameters of the RBM for the optimizer params = list(rbm0.parameters()) + list(rbm1.parameters()) # set up the optimizer and the scheduler opt = torch.optim.SGD(params, lr=1e-3, weight_decay=1e-4) scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=700, gamma=0.94) # build the DBM self.model = DeepBoltzmannMachineLS(rbms=[rbm0, rbm1], optimizer=opt, scheduler=scheduler, learning='PCD', nFantasy=10, ksteps=1)
def __init__(self): super().__init__() layer0 = BernoulliLayer(torch.zeros([1, 1, 28, 28], requires_grad=True)) layer1 = BernoulliLayer(torch.zeros([1, 200], requires_grad=True)) layer2 = BernoulliLayer(torch.zeros([1, 200], requires_grad=True)) interaction0 = InteractionLinear(layer0.bias.shape[1:], layer1.bias.shape[1:]) interaction1 = InteractionLinear(layer1.bias.shape[1:], layer2.bias.shape[1:]) rbm1 = RestrictedBoltzmannMachinePCD(layer0, layer1, interaction0, fantasy_particles=10) rbm2 = RestrictedBoltzmannMachinePCD(layer1, layer2, interaction1, fantasy_particles=10) opt = torch.optim.Adam(chain(rbm1.parameters(), rbm2.parameters()), lr=1e-3) self.model = DeepBeliefNetwork([rbm1, rbm2], opt)
def __init__(self): ''' Builds the RBM with hidden dimension = 200. ''' super().__init__() # initialize the bias with zeros l0bias = torch.zeros([1, 1, 28, 28]) l0bias.requires_grad = True l1bias = torch.zeros([1, 200]) l1bias.requires_grad = True # initialize the bernoulli layers l0 = BernoulliLayer(l0bias) l1 = BernoulliLayer(l1bias) # initialize the interaction layer i0 = InteractionLinear(l0.bias.shape[1:], l1.bias.shape[1:]) # build the RBM rbm0 = RestrictedBoltzmannMachineCD_Smooth(l0, l1, i0, ksteps=1) # get all parameters of the RBM for the optimizer params = list(rbm0.parameters()) # set up the optimizer and the scheduler self.opt = torch.optim.SGD(params, lr=1e-2, weight_decay=1e-5) self.scheduler = torch.optim.lr_scheduler.StepLR(self.opt, step_size=2000, gamma=0.94) self.model = rbm0 #The DBM has a general AIS implementation, which can be used to calculate the log likelihood of the RBM. self.dbm = DeepBoltzmannMachineLS(rbms=[rbm0], optimizer=self.opt, scheduler=self.scheduler, learning='CD', nFantasy=100)