def test_rasterize():
array = np.zeros((64, 64))
d = int(np.sqrt(array.shape[0]))
s = int(np.sqrt(array.shape[1]))
rasterized_array = image._rasterize(array.T,
img_shape=(d, d),
tile_shape=(s, s),
tile_spacing=(1, 1))
assert rasterized_array.shape == (71, 71)
rasterized_array = image._rasterize(array.T,
img_shape=(d, d),
tile_shape=(s, s),
tile_spacing=(1, 1),
scale=False)
assert rasterized_array.shape == (71, 71)
tuple = (np.zeros((64, 64)), np.zeros((64, 64)), np.zeros((64, 64)), None)
rasterized_tuple = image._rasterize(tuple,
img_shape=(d, d),
tile_shape=(s, s),
tile_spacing=(1, 1))
assert rasterized_tuple.shape == (71, 71, 4)
rasterized_tuple = image._rasterize(tuple,
img_shape=(d, d),
tile_shape=(s, s),
tile_spacing=(1, 1),
output=False)
assert rasterized_tuple.shape == (71, 71, 4)
def fit(self, dataset, batch_size=128, epochs=10, frames=6):
"""Fits a new MultFRRBM model.
Args:
dataset (torch.utils.data.Dataset | Dataset): A Dataset object containing the training data.
batch_size (int): Amount of samples per batch.
epochs (list): Number of training epochs per layer.
Returns:
MSE (mean squared error) and log pseudo-likelihood from the training step.
"""
batches = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
collate_fn=collate_fn)
for ep in range(epochs):
logger.info(f'Epoch {ep+1}/{epochs}')
# Resetting epoch's MSE and pseudo-likelihood to zero
mse, pl, cst = 0, 0, 0
inner_trans = tqdm.tqdm(total=len(batches),
desc='Batch',
position=1)
start = time.time()
for ii, batch in enumerate(batches):
x, y = batch
# Checking whether GPU is avaliable and if it should be used
if self.device == 'cuda':
x = x.cuda()
mse2, pl2, cst2 = 0, 0, 0
cost, cost2 = 0, 0
# Initializing the gradient
#self.models[1].optimizer.zero_grad()
for fr in range(frames):
x_ = x[:, fr, :, :].squeeze()
spec_data = fftshift(fftn(x_))[:, :, :, 0]
spec_data = torch.abs(spec_data.squeeze())
spec_data = spec_data.reshape(spec_data.size(0),
self.n_visible)
spec_data = (
(spec_data - torch.mean(spec_data, 0, True)) /
(torch.std(spec_data, 0, True) + c.EPSILON)).detach()
x_ = x_.reshape(x.size(0), self.n_visible)
x_ = ((x_ - torch.mean(x_, 0, True)) /
(torch.std(x_, 0, True) + c.EPSILON)).detach()
# Performs the Gibbs sampling procedure
_, _, _, _, visible_states = self.models[0].gibbs_sampling(
spec_data)
_, _, _, _, visible_states2 = self.models[
1].gibbs_sampling(x_)
# Calculates the loss for further gradients' computation
cost = torch.mean(
self.models[0].energy(spec_data)) - torch.mean(
self.models[0].energy(visible_states))
cost2 = torch.mean(self.models[1].energy(x_)) - torch.mean(
self.models[1].energy(visible_states2))
# Initializing the gradient
self.models[0].optimizer.zero_grad()
self.models[1].optimizer.zero_grad()
# Computing the gradients
#cost /= frames
cost.backward()
#cost2 /= frames
cost2.backward()
# Updating the parameters
self.models[0].optimizer.step()
self.models[1].optimizer.step()
# Detaching the visible states from GPU for further computation
visible_states = visible_states.detach()
visible_states2 = visible_states2.detach()
# Calculating current's batch MSE
batch_mse1 = torch.div(
torch.sum(torch.pow(spec_data - visible_states, 2)),
batch_size).detach()
batch_mse2 = torch.div(
torch.sum(torch.pow(x_ - visible_states2, 2)),
batch_size).detach()
# Calculating the current's batch logarithm pseudo-likelihood
batch_pl1 = self.models[0].pseudo_likelihood(
spec_data).detach()
batch_pl2 = self.models[1].pseudo_likelihood(x_).detach()
# Summing up to epochs' MSE and pseudo-likelihood
mse2 += (batch_mse1 + batch_mse2)
pl2 += (batch_pl1 + batch_pl2)
cst2 += (cost.detach() + cost2.detach())
mse2 /= frames
pl2 /= frames
cst2 /= frames
#cost2 /= frames
#cost2.backward()
#self.models[1].optimizer.step()
mse += mse2
pl += pl2
cst += cst2
if ii % 100 == 99:
print('MSE:', (mse / ii).item(), 'Cost:',
(cst / ii).item())
w8 = self.models[0].W.cpu().detach().numpy()
img = _rasterize(w8.T,
img_shape=(72, 96),
tile_shape=(30, 30),
tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_spec.png')
w8 = self.models[1].W.cpu().detach().numpy()
img = _rasterize(w8.T,
img_shape=(72, 96),
tile_shape=(30, 30),
tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_gauss.png')
x = visible_states[:100].cpu().detach().reshape(
(100, 6912)).numpy()
x = _rasterize(x,
img_shape=(72, 96),
tile_shape=(10, 10),
tile_spacing=(1, 1))
im = Image.fromarray(x)
im = im.convert("LA")
im.save('spectral.png')
x = visible_states2[:100].cpu().detach().reshape(
(100, 6912)).numpy()
x = _rasterize(x,
img_shape=(72, 96),
tile_shape=(10, 10),
tile_spacing=(1, 1))
im = Image.fromarray(x)
im = im.convert("LA")
im.save('sample.png')
inner_trans.update(1)
mse /= len(batches)
pl /= len(batches)
cst /= len(batches)
logger.info(
f'MSE: {mse.item()} | log-PL: {pl.item()} | Cost: {cst.item()}'
)
end = time.time()
self.dump(mse=mse.item(),
pl=pl.item(),
fe=cst.item(),
time=end - start)
return mse, pl, cst
def fit(self, dataset, batch_size=128, epochs=10, frames=6):
"""Fits a new RBM model.
Args:
dataset (torch.utils.data.Dataset): A Dataset object containing the training data.
batch_size (int): Amount of samples per batch.
epochs (int): Number of training epochs.
Returns:
MSE (mean squared error) and log pseudo-likelihood from the training step.
"""
# Transforming the dataset into training batches
batches = DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=workers, collate_fn=collate_fn)
# For every epoch
for e in range(epochs):
logger.info(f'Epoch {e+1}/{epochs}')
# Calculating the time of the epoch's starting
start = time.time()
# Resetting epoch's MSE and pseudo-likelihood to zero
mse, pl, cst = 0, 0, 0
# For every batch
inner = tqdm.tqdm(total=len(batches), desc='Batch', position=1)
for ii, batch in enumerate(batches):
samples, _ = batch
# Checking whether GPU is avaliable and if it should be used
if self.device == 'cuda':
samples = samples.cuda()
mse2, pl2, cst2 = 0, 0, 0
cost = 0
# Initializing the gradient
self.optimizer.zero_grad()
for fr in range(frames):
#torch.autograd.set_detect_anomaly(True)
sps = samples[:, fr, :, :].squeeze()
# Creating the Fourier Spectrum
spec_data = fftshift(fftn(sps))[:,:,:,0]
spec_data = torch.abs(spec_data.squeeze())
# Flattening the samples' batch
spec_data = spec_data.view(spec_data.size(0), self.n_visible)
# Normalizing the samples' batch
spec_data = ((spec_data - torch.mean(spec_data, 0, True)) / (torch.std(spec_data, 0, True) + c.EPSILON)).detach()
# Performs the Gibbs sampling procedure
_, _, _, _, visible_states = self.gibbs_sampling(spec_data)
# Calculates the loss for further gradients' computation
cost += torch.mean(self.energy(spec_data)) - \
torch.mean(self.energy(visible_states))
# Detaching the visible states from GPU for further computation
visible_states = visible_states.detach()
# Gathering the size of the batch
batch_size2 = sps.size(0)
# Calculating current's batch MSE
batch_mse = torch.div(
torch.sum(torch.pow(spec_data - visible_states, 2)), batch_size2).detach()
# Calculating the current's batch logarithm pseudo-likelihood
batch_pl = self.pseudo_likelihood(spec_data).detach()
# Summing up to epochs' MSE and pseudo-likelihood
mse2 += batch_mse
pl2 += batch_pl
cst2 += cost.detach()
# Computing the gradients
cost /= frames
cost.backward()
# Updating the parameters
self.optimizer.step()
mse2 /= frames
pl2 /= frames
cst2 /= frames
mse += mse2
pl += pl2
cst += cst2
if ii % 100 == 99:
print('MSE:', (mse/ii).item(), 'Cost:', (cst/ii).item())
w8 = self.W.cpu().detach().numpy()
img = _rasterize(w8.T, img_shape=(72, 96), tile_shape=(30, 30), tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_spec.png')
x = visible_states[:100].cpu().detach().reshape((100, 6912)).numpy()
x = _rasterize(x, img_shape=(72, 96), tile_shape=(10, 10), tile_spacing=(1, 1))
im = Image.fromarray(x)
im = im.convert("LA")
im.save('spectral.png')
inner.update(1)
# Normalizing the MSE and pseudo-likelihood with the number of batches
mse /= len(batches)
pl /= len(batches)
cst /= len(batches)
# Calculating the time of the epoch's ending
end = time.time()
# Dumps the desired variables to the model's history
self.dump(mse=mse.item(), pl=pl.item(), fe=cst.item(), time=end-start)
logger.info(f'MSE: {mse} | log-PL: {pl} | Cost: {cst}')
self.p = 0
return mse, pl, cst
def fit(self, dataset, batch_size=128, epochs=10, frames=6):
"""Fits a new DBM model.
Args:
dataset (torch.utils.data.Dataset): A Dataset object containing the training data.
batch_size (int): Amount of samples per batch.
epochs (int): Number of training epochs.
frames (int): Number of frames per video clip.
Returns:
MSE (mean squared error) from the training step.
"""
# Transforming the dataset into training batches
batches = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True,
num_workers=workers,
collate_fn=collate_fn)
# For every epoch
for e in range(epochs):
logger.info(f'Epoch {e+1}/{epochs}')
# Calculating the time of the epoch's starting
start = time.time()
# Resetting epoch's MSE and pseudo-likelihood to zero
mse, pl, cst = 0, 0, 0
ii = 1
#cst = torch.zeros((self.n_layers), dtype=torch.float, device=self.device)
# For every batch
for samples, y in tqdm(batches):
#cost = 0
cst2 = 0
#cost = torch.zeros((self.n_layers), dtype=torch.float, device=self.device)
#cst2 = torch.zeros((self.n_layers), dtype=torch.float, device=self.device)
if self.device == 'cuda':
samples = samples.cuda()
mse2, pl2 = 0, 0
for fr in range(frames):
torch.autograd.set_detect_anomaly(True)
# Flattening the samples' batch
sps = samples[:, fr, :, :]
sps = sps.view(sps.size(0), self.n_visible)
# Normalizing the samples' batch
sps = ((sps - torch.mean(sps, 0, True)) /
(torch.std(sps, 0, True) + 10e-6)).detach()
# Performs the fast inference
mf = self.fast_infer(sps)
# Performs the mean-field approximation
mf = self.mean_field(mf)
# Performs the Gibbs sampling procedure
visible_states = self.gibbs_sampling(sps)
# Calculates the loss for further gradients' computation
for idx, model in enumerate(self.models):
# Initializing the gradient
model.optimizer.zero_grad()
cost = torch.mean(model.energy(mf[idx])) - \
torch.mean(model.energy(visible_states[idx]))
cst2 += cost.detach().item()
# Computing the gradients
cost.backward()
# Updating the parameters
model.optimizer.step()
# Gathering the size of the batch
batch_size2 = sps.size(0)
# Calculating current's batch MSE
batch_mse = torch.div(
torch.sum(torch.pow(sps - visible_states[0], 2)),
batch_size2).detach()
# Calculating the current's batch logarithm pseudo-likelihood
#batch_pl = self.pseudo_likelihood(sps).detach()
# Summing up to epochs' MSE and pseudo-likelihood
mse2 += batch_mse
#pl2 += batch_pl
#cst2 += cost.detach()
mse2 /= frames
cst2 /= (frames * self.n_layers)
#pl2 /= frames
#cst2 /= frames
mse += mse2
#pl += pl2
cst += cst2
if ii % 100 == 99:
print('MSE:', (mse / ii).item(), 'Cost:', cst / ii)
w8 = self.models[0].W.cpu().detach().numpy()
img = _rasterize(w8.T,
img_shape=(self.visible_shape[0],
self.visible_shape[1]),
tile_shape=(30, 30),
tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_fit_ucf101.png')
ii += 1
# Normalizing the MSE and pseudo-likelihood with the number of batches
mse /= len(batches)
#pl /= len(batches)
cst /= len(batches)
# Calculating the time of the epoch's ending
end = time.time()
# Dumps the desired variables to the model's history
#self.dump(mse=mse.item(), pl=pl.item(), fe=cst.item(), time=end-start)
self.dump(mse=mse.item(), fe=cst, time=end - start)
#logger.info(f'MSE: {mse} | log-PL: {pl} | Cost: {cst}')
logger.info(f'MSE: {mse} | Cost: {cst}')
return mse #,pl, cst
def fit(self, dataset, batch_size=128, epochs=10, frames=6):
"""Fits a new RBM model.
Args:
dataset (torch.utils.data.Dataset): A Dataset object containing the training data.
batch_size (int): Amount of samples per batch.
epochs (int): Number of training epochs.
Returns:
MSE (mean squared error) and log pseudo-likelihood from the training step.
"""
# Transforming the dataset into training batches
batches = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=workers,
collate_fn=collate_fn)
# For every epoch
for e in range(epochs):
logger.info(f'Epoch {e+1}/{epochs}')
#if e > 1:
# self.momentum = 0.9
# Calculating the time of the epoch's starting
start = time.time()
# Resetting epoch's MSE and pseudo-likelihood to zero
mse, pl, cst = 0, 0, 0
# For every batch
inner = tqdm.tqdm(total=len(batches), desc='Batch', position=1)
for ii, batch in enumerate(batches):
samples, _ = batch
# Checking whether GPU is avaliable and if it should be used
if self.device == 'cuda':
samples = samples.cuda()
#dy = samples.size(2) #72
#dx = samples.size(3) #96 x(72, 96)
#max_value = samples.max()
for fr in range(1, frames):
samples[:, 0, :, :] -= samples[:, fr, :, :]
#samples[:, 0, :, :] /= (max_value*frames)
samples = samples[:, 0, :, :].reshape(
len(samples),
samples.size(2) * samples.size(3))
samples = ((samples - torch.mean(samples, 0, True)) /
(torch.std(samples, 0, True) + 1e-6)).detach()
mse2, pl2, cst2 = 0, 0, 0
# Performs the Gibbs sampling procedure
_, _, _, _, visible_states = self.gibbs_sampling(samples)
# Calculates the loss for further gradients' computation
cost = torch.mean(self.energy(samples)) - \
torch.mean(self.energy(visible_states))
# Initializing the gradient
self.optimizer.zero_grad()
# Computing the gradients
cost.backward()
# Updating the parameters
self.optimizer.step()
# Gathering the size of the batch
batch_size2 = samples.size(0)
# Calculating current's batch MSE
batch_mse = torch.div(
torch.sum(torch.pow(samples - visible_states, 2)),
batch_size2).detach()
# Calculating the current's batch logarithm pseudo-likelihood
batch_pl = self.pseudo_likelihood(samples).detach()
# Detaching the visible states from GPU for further computation
visible_states = visible_states.detach()
# Summing up to epochs' MSE and pseudo-likelihood
mse2 += batch_mse
pl2 += batch_pl
cst2 += cost.detach()
#mse2 /= frames
#pl2 /= frames
#cst2 /= frames
mse += mse2
pl += pl2
cst += cst2
if ii % 100 == 99:
print('MSE:', (mse / ii).item(), 'Cost:',
(cst / ii).item())
#w8 = self.W.cpu().detach().numpy()[1:, :]
w8 = self.W.cpu().detach().numpy()
img = _rasterize(w8.T,
img_shape=(72, 96),
tile_shape=(30, 30),
tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_spec_.png')
inner.update(1)
# Normalizing the MSE and pseudo-likelihood with the number of batches
mse /= len(batches)
pl /= len(batches)
cst /= len(batches)
# Calculating the time of the epoch's ending
end = time.time()
# Dumps the desired variables to the model's history
self.dump(mse=mse.item(),
pl=pl.item(),
fe=cst.item(),
time=end - start)
logger.info(f'MSE: {mse} | log-PL: {pl} | Cost: {cst}')
self.p = 0
return mse, pl, cst
def fit(self, dataset, batch_size=128, epochs=1, frames=6):
"""Fits a new E-drop RBM model.
Args:
dataset (torch.utils.data.Dataset): A Dataset object containing the training data.
batch_size (int): Amount of samples per batch.
epochs (int): Number of training epochs.
frames (int): Number of frames.
Returns:
MSE (mean squared error) and(or not) log pseudo-likelihood from the training step.
"""
# Transforming the dataset into training batches
batches = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
drop_last=True,
num_workers=workers,
collate_fn=collate_fn)
# For every epoch
for e in range(epochs):
logger.info(f'Epoch {e+1}/{epochs}')
# Calculating the time of the epoch's starting
start = time.time()
# Resetting epoch's MSE and pseudo-likelihood to zero
mse, pl, cst = 0, 0, 0
ii = 1
# For every batch
for samples, y in tqdm(batches):
cost = 0
if self.device == 'cuda':
samples = samples.cuda()
mse2, pl2, cst2 = 0, 0, 0
for fr in range(frames):
# Returns the Energy-based Dropout mask to one
self.M = torch.ones((batch_size, self.n_hidden),
device=self.device)
#torch.autograd.set_detect_anomaly(True)
# Flattening the samples' batch
sps = samples[:, fr, :, :]
sps = sps.view(sps.size(0), self.n_visible)
# Normalizing the samples' batch
sps = ((sps - torch.mean(sps, 0, True)) /
(torch.std(sps, 0, True) + c.EPSILON)).detach()
# Performs the initial Gibbs sampling procedure (pre-dropout)
pos_hidden_probs, pos_hidden_states, neg_hidden_probs, neg_hidden_states, visible_states = \
self.gibbs_sampling(sps)
# Calculating energy of positive phase sampling
e = self.total_energy(pos_hidden_states, sps)
# Calculating energy of negative phase sampling
e1 = self.total_energy(neg_hidden_states, visible_states)
# Performing the energy-based dropout
self.energy_dropout(e1 - e, pos_hidden_probs,
neg_hidden_probs)
# Performs the post Gibbs sampling procedure (post-dropout)
_, _, _, _, visible_states = self.gibbs_sampling(sps)
# Detaching the visible states from GPU for further computation
visible_states = visible_states.detach()
# Calculates the loss for further gradients' computation
cost += torch.mean(self.energy(sps)) - \
torch.mean(self.energy(visible_states))
# Calculating current's batch MSE
batch_mse = torch.div(
torch.sum(torch.pow(sps - visible_states, 2)),
batch_size)
# Calculating the current's batch logarithm pseudo-likelihood
batch_pl = self.pseudo_likelihood(sps).detach()
# Summing up to epochs' MSE and pseudo-likelihood
mse2 += batch_mse
pl2 += batch_pl
cst2 += cost.detach()
# Initializing the gradient
self.optimizer.zero_grad()
# Computing the gradients
cost /= frames
cost.backward()
# Updating the parameters
self.optimizer.step()
mse2 /= frames
pl2 /= frames
cst2 /= frames
mse += mse2
pl += pl2
cst += cst2
if ii % 100 == 99:
print('MSE:', (mse / ii).item(), 'Cost:',
(cst / ii).item())
w8 = self.W.cpu().detach().numpy()
img = _rasterize(w8.T,
img_shape=(72, 96),
tile_shape=(30, 30),
tile_spacing=(1, 1))
im = Image.fromarray(img)
im.save('w8_edp_ucf101.png')
ii += 1
# Normalizing the MSE and pseudo-likelihood with the number of batches
mse /= len(batches)
cst /= len(batches)
pl /= len(batches)
# Calculating the time of the epoch's ending
end = time.time()
# Dumps the desired variables to the model's history
#self.dump(mse=mse.item(), pl=pl.item(), time=end-start)
self.dump(mse=mse.item(),
pl=pl.item(),
fe=cst.item(),
time=end - start)
#logger.info('MSE: %f | log-PL: %f', mse, pl)
logger.info(f'MSE: {mse} | Cost: {cst}')
return mse, pl, cst