def find_centers(self, FV):
N = FV.shape[0]
assert N > 1
center_list = [FV[0, :]]
histogram_of_point_list = [1]
for n in range(1, N):
f_n = FV[n:(n + 1), :]
c_n = torch.vstack(center_list)
d_n = torch.cdist(f_n,
c_n,
p=2,
compute_mode="donot_use_mm_for_euclid_dist")
assert d_n.shape[0] == 1
value_min, arg_min = torch.min(d_n, dim=1)
if value_min <= self.threshold_distance:
center_old = center_list[arg_min]
wc = histogram_of_point_list[arg_min]
center_new = ((wc * center_old) + f_n) / (wc + 1)
center_list[arg_min] = center_new
histogram_of_point_list[arg_min] = wc + 1
else:
center_list.append(f_n)
histogram_of_point_list.append(1)
if len(histogram_of_point_list) == 1:
center = center_list[0].view(1, -1)
else:
center = torch.vstack(center_list)
return (center, histogram_of_point_list)
def estimate(x, y, z, epoch=100):
perm = torch.randperm(x.shape[0])
x, y, z = x[perm], y[perm], z[perm]
# split train, eval
batch_size = x.shape[0] * 2 // 3
x, x_eval = torch.split(x, batch_size)
y, y_eval = torch.split(y, batch_size)
z, z_eval = torch.split(z, batch_size)
# xyz batch data
data_batch = torch.vstack(make_batch(x, torch.hstack((y, z))))
label_batch = torch.vstack((torch.ones((batch_size, 1)), torch.zeros((batch_size, 1))))
# estimate (x|yz)
estimator_xyz = Estimator(x.shape[1] + y.shape[1] + z.shape[1])
for _ in range(epoch):
estimator_xyz.train_batch(data_batch, label_batch)
xyz_i = estimator_xyz.estimate_divergence([make_batch(x_eval, torch.hstack((y_eval, z_eval)))])
# estimate (x|z)
data_batch = torch.vstack(make_batch(x, z))
estimator_xz = Estimator(x.shape[1] + z.shape[1])
for _ in range(epoch):
estimator_xz.train_batch(data_batch, label_batch)
xz_i = estimator_xz.estimate_divergence([make_batch(x_eval, z_eval)])
return xyz_i - xz_i
def _posterior_decide_with_softmax_logits(self, *logits):
'''
使用多个feature分别forward之后生成的softmax logits进行联合决策。
基本方法是：所有样本先在每个feature内选出一个概率值最高的结果，
然后各个feature之间最高的概率值再进行比较来决出最高的概率值，
并且以该最高概率值对应的标签作为联合决策结果返回
'''
m_logit_list = []
m_label_list = []
for logit in logits:
# 对于每一个feature的softmax logits，先取出每一个样本的最大值和最大值对应的下标
max_item = torch.max(logit, dim=1)
m_logit_list.append(max_item.values)
m_label_list.append(max_item.indices)
logit_tensor = torch.vstack(m_logit_list).transpose(
0, 1).contiguous().cuda()
label_tensor = torch.vstack(m_label_list).transpose(
0, 1).contiguous().cuda()
# 对所有feature的逐样本最大logit，找出最大的logit对应的feature
label_indexes = torch.max(logit_tensor, dim=1, keepdim=True).indices
# 返回最大logit的feature的最大值的label
ret_labels = torch.gather(label_tensor, 1, label_indexes)
return ret_labels.squeeze()
def get_pseudo_labels(self):
matrix = torch.tensor([], device=self.device)
# Iterate through unlabeled loader
for batch_idx, (data, target, idx) in enumerate(self.unlabeled_loader_original):
with torch.no_grad():
# Get predictions for unlabeled samples
out = self.model(data.to(self.device))
p_out = torch.softmax(out, dim=1) # turn into probability distribution
confidence, pseudo_lbl = torch.max(p_out, dim=1)
pseudo_lbl_batch = torch.vstack((idx.to(self.device), confidence, pseudo_lbl)).T
# Append to matrix
matrix = torch.cat((matrix, pseudo_lbl_batch), dim=0) # (n_unlabeled, 3)
n_unlabeled = matrix.shape[0]
indices = matrix[:, 0].cpu().numpy().astype(int)
ground_truth = self.targets_list[indices]
matrix = torch.vstack((matrix.T, torch.tensor(ground_truth, device=self.device))).T # (n_unlabeled, 4)
matrix = torch.vstack((matrix.T, torch.zeros(n_unlabeled, device=self.device))).T # (n_unlabeled, 5)
# matrix columns: [index, confidence, pseudo_label, true_label, is_ground_truth]
# Check if pseudo label is ground truth
for i in range(n_unlabeled):
if matrix[i, 2] == matrix[i, 3]:
matrix[i, 4] = 1
return matrix
def run_batch_size_test(self, device):
# Runs test for reproducibility across batch sizes on a device
voice256 = Voice(SynthConfig(batch_size=256)).to(device)
out256 = voice256(0)
voice128 = Voice(SynthConfig(batch_size=128)).to(device)
out128 = torch.vstack([voice128(0), voice128(1)])
voice64 = Voice(SynthConfig(batch_size=64)).to(device)
out64 = torch.vstack([voice64(0), voice64(1), voice64(2), voice64(3)])
voice32 = Voice(SynthConfig(batch_size=32)).to(device)
out32 = torch.vstack([
voice32(0),
voice32(1),
voice32(2),
voice32(3),
voice32(4),
voice32(5),
voice32(6),
voice32(7),
])
# TODO there are some unexpected, very small numerical
# errors between some, but not all, batch sizes
# See https://github.com/torchsynth/torchsynth/issues/326
assert torch.all(torch.isclose(out256, out128))
assert torch.all(torch.isclose(out256, out64))
assert torch.all(torch.isclose(out256, out32))
def unit_matrix(self, index, frame):
"""
(x+Tx) / (z+Tz) = X
(y+Ty) / (z+Tz) = Y
x-zX = -Tx+XTz
y-zY = -Ty+YTz
|-1 0 X| |Tx| |x-zX|
|0 -1 Y| |Ty| = |y-zY|
|Tz|
"""
input_3d = self.inputs_3d[index, frame].squeeze(0) #(17,3)
input_2d = self.inputs_2d[index, frame].squeeze(0) #(17,2)
J = input_3d.shape[0]
x = input_3d[:, 0]
y = input_3d[:, 1]
z = input_3d[:, 2]
X = input_2d[:, 0]
Y = input_2d[:, 1]
I = torch.ones_like(X)
O = torch.zeros_like(X)
unit_A = torch.hstack(
[torch.vstack([-I, O, X]),
torch.vstack([O, -I, Y])]).transpose(1, 0)
unit_b = torch.hstack([x - z * X, y - z * Y])
return unit_A, unit_b
def random_pairs_of_minibatches(args, minibatches):
ld = len(minibatches)
pairs = []
tdlist = np.arange(ld)
txlist = np.arange(args.batch_size)
for i in range(ld):
for j in range(args.batch_size):
(tdi, tdj), (txi, txj) = np.random.choice(
tdlist, 2, replace=False), np.random.choice(txlist,
2,
replace=True)
if j == 0:
xi, yi, di = torch.unsqueeze(
minibatches[tdi][0][txi],
dim=0), minibatches[tdi][1][txi], minibatches[tdi][2][txi]
xj, yj, dj = torch.unsqueeze(
minibatches[tdj][0][txj],
dim=0), minibatches[tdj][1][txj], minibatches[tdj][2][txj]
else:
xi, yi, di = torch.vstack(
(xi, torch.unsqueeze(
minibatches[tdi][0][txi], dim=0))), torch.hstack(
(yi, minibatches[tdi][1][txi])), torch.hstack(
(di, minibatches[tdi][2][txi]))
xj, yj, dj = torch.vstack(
(xj, torch.unsqueeze(
minibatches[tdj][0][txj], dim=0))), torch.hstack(
(yj, minibatches[tdj][1][txj])), torch.hstack(
(dj, minibatches[tdj][2][txj]))
pairs.append(((xi, yi, di), (xj, yj, dj)))
return pairs
def performance(Ts: dict, files: list) -> None:
"""Prints performance of thresholds for train and/or test inputs"""
if opt.train is not None:
print("TRAIN PERFORMANCE")
train_labs, train_preds, train_distances = files[opt.train]
known_labs, known_preds, known_distances = files[opt.known]
_, unknown_preds, unknown_distances = files[opt.unknown]
# combine opt.train, opt.known, opt.unknown
preds = torch.vstack([train_preds, known_preds,
unknown_preds]).long()[:, 0]
labs = torch.cat([
train_labs, known_labs,
torch.ones(unknown_preds.shape[0], dtype=int).to(0) * -1
]).long()
distances = torch.cat(
[train_distances, known_distances, unknown_distances])[:, 0]
print_performance(Ts, labs, preds, distances)
if opt.test is not None:
print("TEST PERFORMANCE")
test_labs, test_preds, test_distances = files[opt.test]
_, test_unknown_preds, test_unknown_distances = files[opt.test_unknown]
# combine opt.test, opt.test_unknown
preds = torch.vstack([test_preds, test_unknown_preds]).long()[:, 0]
labs = torch.cat([
test_labs,
torch.ones(test_unknown_preds.shape[0], dtype=int).to(0) * -1
]).long()
distances = torch.cat([test_distances, test_unknown_distances])[:, 0]
print_performance(Ts, labs, preds, distances)
def get_grad(input):
grad = input[1:] - input[:-1]
zeros = torch.Tensor([[0, 0, 0]]).to(device)
tmp1 = torch.vstack((grad, zeros))
tmp2 = torch.vstack((zeros, grad))
grad = (tmp1 + tmp2) / 2
return grad
def knn(labs: torch.Tensor, preds: torch.Tensor, distances: torch.Tensor,
avs: torch.Tensor, outputs: torch.Tensor, targets: torch.Tensor,
train_avs: torch.Tensor, train_labs: torch.Tensor, K: int, start: int) -> tuple:
"""
Update input labels and activation vectors with those of preds and outputs. Also update
input distances with distances of
"""
# update existing labels and activation vectors
if labs is not None:
labs = torch.cat([labs, targets])
avs = torch.vstack([avs, outputs])
else:
labs = targets.clone()
avs = outputs.clone()
# update distances with distances of new activation vectors
for av in outputs:
dist = torch.norm(train_avs - av, dim=1, p=None) # distances to all lookup table entries
nearest = dist.topk(K, largest=False) # K nearest labels
if preds is not None:
# update predictions (nearest K labels)
preds = torch.vstack([preds, train_labs[nearest.indices[start:]]])
distances = torch.vstack([distances, nearest.values[start:]]) # update distances
else:
# init predictions and distances
preds = train_labs[nearest.indices[start:]].clone()
distances = nearest.values[start:].clone()
return labs, preds, distances, avs
def test_noise(self):
# Here we create there noise modules with different batch sizes.
# We each noise module a number of times to obtain an equal number
# of noise signals. All these noise samples should equal each other.
# i.e., noise should be returned deterministically regardless of the
# batch size.
synthconfig32 = SynthConfig(32)
synthconfig64 = SynthConfig(64)
synthconfig128 = SynthConfig(128)
noise32 = synthmodule.Noise(synthconfig32, seed=0)
noise64 = synthmodule.Noise(synthconfig64, seed=0)
noise128 = synthmodule.Noise(synthconfig128, seed=0)
# A different seed should give a different result
noise128_diff = synthmodule.Noise(synthconfig128, seed=42)
out1 = torch.vstack((noise32(), noise32(), noise32(), noise32()))
out2 = torch.vstack((noise64(), noise64()))
out3 = noise128()
out4 = noise128_diff()
assert torch.all(out1 == out2)
assert torch.all(out2 == out3)
assert torch.all(out3 != out4)
with pytest.raises(ValueError):
# If the batch size is not a multiple of BASE_REPRODUCIBLE_BATCH_SIZE,
# it should raise an error in reproducible mode
synthconfigwrong = SynthConfig(BASE_REPRODUCIBLE_BATCH_SIZE + 1)
synthmodule.Noise(synthconfigwrong, seed=0)
def run_once(timer_msg, outfn=None):
with Timer(timer_msg):
fit_n_iters_list, weights_list, biases_list = runner(prob_it())
if outfn:
weights = torch.vstack(weights_list)
biases = torch.vstack(biases_list)[:, 0]
torch.save((fit_n_iters_list, weights, biases), outfn)
def do_oversampling(N_cluster, cluster_label, X, OUT, over_coef): """ This function will use oversampling to prevent from overfitting Overfitting can happen when training data got stacked in a very densed region Oversampling will then be done in the region where cluster size is small Input: N_cluster: number of cluster estimated by Gap Statistics cluster_label: label assigned to every point X: Training data for the input layer OUT: Training data for the output layer over_coef: this will be used in the oversampling method to increase number of points in the less densed cluster region Output: X_over: Training data for the input layer that has been oversampled OUT_over: Training data for the output layer that has been oversampled """ X_over = torch.clone(X) OUT_over = torch.clone(OUT) for cluster in range(N_cluster): idx = torch.where(cluster_label==cluster)[0] X_cluster = torch.index_select(X, 0, idx) OUT_cluster = torch.index_select(OUT, 0, idx) for counter in range(over_coef[cluster]-1): X_over = torch.vstack((X_over, X_cluster)) OUT_over = torch.vstack((OUT_over, OUT_cluster)) return X_over, OUT_over
def get_batch(self, batch_indices, labels_important: bool):
X = torch.vstack(
[self.data_reading_method(self.data[i]) for i in batch_indices])
y = torch.vstack(
[self.label_reading_method(self.labels[i]) for i in batch_indices])
semi_supervision_mask = torch.ones(y.shape)
if labels_important:
# Fill in with semi-supervision labels
for j, label in enumerate(y):
instance_index = batch_indices[j]
if self.index.labelled_idx[instance_index]:
pass
elif self.index.temp_labelled_idx[instance_index]:
y[j] = self.temp_labels[instance_index]
elif self.index.unlabelled_idx[instance_index]:
y[j] = torch.exp(
torch.tensor(self.last_preds[instance_index]))
semi_supervision_mask[j] = self.semi_supervision_multiplier
else:
raise Exception(
"Instance index does not appear in any of the annotation status lists"
)
return X, y, torch.tensor([]), semi_supervision_mask
return (
X,
torch.tensor([]),
torch.tensor([]), # was lengths
semi_supervision_mask,
)
def prepare_data(datasets: list,
featureList: list,
train_ratio: float = 0.8,
val_ratio: float = 0.2,
test_size: int = 100,
SEED: int = 2021):
# process data
data = [func.processData(d, featureList, shutdown=False) for d in datasets]
input_data = [np.vstack(d) for d in data]
x_tensors = [
func.normaliseT(torch.from_numpy(x).float()) for x in input_data
]
y_tensors = [torch.from_numpy(x).float() for x in input_data]
# prepare datasets
test_sets = [(x_tensor[-test_size:], y_tensor[-test_size:])
for x_tensor, y_tensor in zip(x_tensors, y_tensors)]
x_training = torch.vstack(
[x_tensor[:-test_size] for x_tensor in x_tensors])
y_training = torch.vstack(
[y_tensor[:-test_size] for y_tensor in y_tensors])
dataset = TensorDataset(x_training, y_training)
N = len(x_training)
train_ratio = int(train_ratio * N)
val_ratio = int(val_ratio * N)
print("Train: ", train_ratio, ", Validation: ", val_ratio)
train_set, val_set = random_split(
dataset, [train_ratio, val_ratio],
generator=torch.Generator().manual_seed(SEED))
return train_set, val_set, test_sets
def melt(x, context, chunk_size, sortby=6):
days = context[:, 0]
sides = x[:, 0]
melted_x = []
melted_c = []
for d in torch.unique(days):
for s in np.array([-1, +1]):
grouped_x = x[(days == d.item()) & (sides == s)]
grouped_c = context[(days == d.item()) & (sides == s)]
if len(grouped_x) > 1:
sorted_idx = grouped_c[:, sortby].sort()[1]
chunks_x = torch.split(grouped_x[sorted_idx],
chunk_size,
dim=0)
chunks_c = torch.split(grouped_c[sorted_idx],
chunk_size,
dim=0)
for chidx in np.arange(len(chunks_x)):
melted_x.append(torch.mean(chunks_x[chidx], dim=0))
melted_c.append(torch.mean(chunks_c[chidx], dim=0))
elif len(grouped_x) == 1:
melted_x.append(grouped_x)
melted_c.append(grouped_c)
return torch.vstack(melted_x), torch.vstack(melted_c)
def sample(self,
sample_shape: Shape = torch.Size(),
x: Optional[Tensor] = None,
**kwargs) -> Tensor:
r"""Return samples from posterior ensemble.
The samples are drawn according to their assigned weight. The number of samples
for each distributino is drawn from a corresponding multinomial distribution.
Then each component posterior is sampled individually and all samples are
aggregated afterwards.
All kwargs are passed directly through to `posterior.sample()`.
Args:
sample_shape: Desired shape of samples that are drawn from posterior
ensemble. If sample_shape is multidimensional we simply draw
`sample_shape.numel()` samples and then reshape into the desired shape.
x: Conditioning context. If none is provided and no default context is set,
an error will be raised.
Returns:
Samples drawn from the ensemble distribution.
"""
num_samples = torch.Size(sample_shape).numel()
posterior_indizes = torch.multinomial(self._weights,
num_samples,
replacement=True)
samples = []
for posterior_index, sample_size in torch.vstack(
posterior_indizes.unique(return_counts=True)).T:
sample_shape_c = torch.Size((int(sample_size), ))
samples.append(self.posteriors[posterior_index].sample(
sample_shape_c, x=x, **kwargs))
return torch.vstack(samples).reshape(*sample_shape, -1)
def sample(self, batch_size):
state, action, next_state, reward = zip(
*random.sample(self.memory, batch_size))
#return random.sample(self.memory, batch_size)
return torch.vstack(state), torch.tensor(action).to(device).view(
-1,
1), torch.vstack(next_state), torch.Tensor(reward).to(device).view(
-1, 1)
def optimize_model():
if len(memory) <= BATCH_SIZE:
return
transition = memory.sample(BATCH_SIZE)
batch = Transition(*zip(*transition))
non_final_mask = torch.tensor(tuple(
map(lambda s: s is not None, batch.next_state)),
device=device,
dtype=torch.bool)
non_final_next_states = {
'p':
torch.vstack([s['p'] for s in batch.next_state if s is not None]),
'r':
torch.vstack([s['r'] for s in batch.next_state if s is not None]),
'oo':
torch.vstack([s['oo'] for s in batch.next_state if s is not None]),
'c':
torch.vstack([s['c'] for s in batch.next_state if s is not None]),
'mem':
torch.vstack([s['mem'] for s in batch.next_state if s is not None])
}
state_batch = {
'p': torch.vstack([s['p'] for s in batch.state if s is not None]),
'r': torch.vstack([s['r'] for s in batch.state if s is not None]),
'oo':
torch.vstack([s['oo'] for s in batch.state if s is not None]),
'c': torch.vstack([s['c'] for s in batch.state if s is not None]),
'mem':
torch.vstack([s['mem'] for s in batch.state if s is not None])
}
action_batch = torch.cat(batch.action)
reward_batch = torch.cat(batch.reward)
state_action_values = policy_net.forward(state_batch,
opti=True).gather(
1, action_batch)
next_state_values = torch.zeros(BATCH_SIZE,
device=device,
dtype=torch.double)
action_batch = action_batch[non_final_mask]
temp = target_net.forward(non_final_next_states,
opti=True).gather(1, action_batch)
next_state_values[non_final_mask] = temp.view(1, -1)
expected_state_action_values = (next_state_values *
GAMMA) + reward_batch
loss = nn.functional.mse_loss(
state_action_values, expected_state_action_values.unsqueeze(1))
policy_net.optim.zero_grad()
loss.backward()
policy_net.optim.step()
def test_higher_dim(self) -> None:
seq_lens = random.sample(range(5, 20), 3)
tss = [[torch.rand(l) for l in seq_lens] for _ in range(10)]
src = torch.vstack([torch.hstack(ts) for ts in tss])
index = torch.repeat_interleave(torch.tensor(seq_lens))
out = scatter_softmax(src, index, dim=1)
expected = torch.vstack(
[torch.hstack([F.softmax(t, 0) for t in ts]) for ts in tss])
self.assertTrue(torch.allclose(out, expected))
def collate_fn(data_list: List[Data]):
batch = torch.vstack([data.pos for data in data_list])
batch = batch.reshape(-1, *data_list[0].pos.shape).double()
pose = torch.vstack([data.pose for data in data_list])
if onehot:
pose = torch.nn.functional.one_hot(pose.flatten(),
num_classes=10)
else:
pose = pose.reshape(-1, *data_list[0].pose.shape).double()
return BatchWrapper(x=batch, pose=pose)
def get_regularized_slice_dist_matrix(x,y,z,reg):
points1 = torch.vstack((torch.arange(x, device=device, dtype=torch.float32),torch.zeros(x, device=device))).T
dist_matrix1 = (torch.cdist(points1,points1)/(x-1))**2
K1 = torch.divide(dist_matrix1, -reg)
torch.exp(K1, out=K1)
points2 = torch.vstack((torch.arange(y, device=device, dtype=torch.float32),torch.zeros(y, device=device))).T
dist_matrix2 = (torch.cdist(points2,points2)/(y-1))**2
K2 = torch.divide(dist_matrix2, -reg)
torch.exp(K2, out=K2)
return K1,K2
def forward(self, W_query, W_supports):
"""
Find scores of each token being start and end token for an entity.
Args:
W_query (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of query sequence tokens in the vocabulary.
W_supports (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of support sequence tokens in the vocabulary.
Returns:
p_start (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Scores of each token as
being start token of an entity
p_end (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Scores of each token as
being end token of an entity
"""
support_sizes = W_supports["sizes"].tolist()
start_token_id = W_supports["start_token_id"].item()
end_token_id = W_supports["end_token_id"].item()
del W_supports["sizes"]
del W_supports["start_token_id"]
del W_supports["end_token_id"]
q = self.BERT(**W_query)
S = self.BERT(**W_supports)
p_starts = None
p_ends = None
start_token_masks = W_supports["input_ids"] == start_token_id
end_token_masks = W_supports["input_ids"] == end_token_id
for i, size in enumerate(support_sizes):
if i == 0:
s = 0
else:
s = support_sizes[i - 1]
s_start = S[s : s + size][start_token_masks[s : s + size]]
s_end = S[s : s + size][end_token_masks[s : s + size]]
p_start = torch.matmul(q[i], s_start.T).sum(1).softmax(0)
p_end = torch.matmul(q[i], s_end.T).sum(1).softmax(0)
if p_starts is not None:
p_starts = torch.vstack((p_starts, p_start))
p_ends = torch.vstack((p_ends, p_end))
else:
p_starts = p_start
p_ends = p_end
return p_starts, p_ends
def __call__(self, batch):
graph_list = []
data_list = []
label_list = []
for frame in batch:
graph_list.append(copy.deepcopy(self.graph))
data_list.append(torch.from_numpy(frame[0]))
label_list.append(torch.from_numpy(frame[1]))
graph_batch = dgl.batch(graph_list)
data_batch = torch.vstack(data_list)
label_batch = torch.vstack(label_list)
return graph_batch, data_batch, label_batch
def collate(graph_entries: List[MoleculeGraphEntry]):
graphs, features, labels = zip(*graph_entries)
batched_graph = dgl.batch(graphs)
batched_features = torch.vstack(features)
batched_labels = {}
for label_name in labels[0]:
batched_labels[label_name] = torch.vstack(
[label[label_name].reshape(-1, 1) for label in labels]
)
return batched_graph, batched_features, batched_labels
def test(epoch, net1):
net1.eval()
true = list()
pred = list()
with torch.no_grad():
for batch_idx, (inputs, targets, _) in enumerate(test_loader):
inputs, targets = inputs.cuda(), targets.cuda()
outputs1 = net1(inputs)
true.append(targets)
pred.append(outputs1)
true = torch.vstack(true)
pred = torch.vstack(pred)
rmse = torch.sqrt(F.mse_loss(true, pred, reduction='none').mean(dim=0))
pcc = utils.PCC(true, pred)
print("\n| Test Epoch #{}\t Arr RMSE: {}\n Arr PCC: {}\n".format(epoch, rmse[0], pcc[0]))
def discriminator_train_on_batch(self, real, fake):
x_real, y_real = real
x_fake, y_fake = fake
x = torch.vstack([x_real, x_fake])
y = torch.vstack([y_real, y_fake])
self.discriminator.zero_grad()
output = self.discriminator(x)
loss, losses = self.discriminator_loss(y, output)
loss.backward()
self.d_optimizer.step()
metrics = self.discriminator_metrics(y, output)
return {**losses, **metrics}
def test_model():
from nsgp import NSGP
from nsgp.utils.inducing_functions import f_kmeans
import torch
import numpy as np
num_low = 25
num_high = 25
gap = -.1
X = torch.vstack((torch.linspace(-1, -gap/2.0, num_low)[:, np.newaxis],
torch.linspace(gap/2.0, 1, num_high)[:, np.newaxis])).reshape(-1, 1)
y = torch.vstack((torch.zeros((num_low, 1)), torch.ones((num_high, 1))))# + torch.rand(num_high+num_low, 1)
scale = torch.sqrt(y.var())
offset = y.mean()
y = ((y-offset)/scale).reshape(-1, 1)
X_new = torch.linspace(-1, 1, 100).reshape(-1, 1)
X_bar = f_kmeans(X, num_inducing_points=5)#, random_state=0)
model = NSGP(X, y, X_bar=X_bar, jitter=10**-5)#, random_state=0)
optim = torch.optim.Adam(model.parameters(), lr=0.1)
# optim = torch.optim.SGD(model.parameters(), lr=0.01)
losses = []
model.train()
for _ in range(200):
optim.zero_grad()
loss = model()
losses.append(loss.item())
loss.backward()
optim.step()
print(losses)
model.eval()
with torch.no_grad():
y_new, y_var = model.predict(X_new)
y_std2 = y_var.diagonal()**0.5 * 2
fig, ax = plt.subplots(3, 1, figsize=(10, 16))
ax[0].scatter(X, y)
ax[0].plot(X_new.numpy(), y_new.numpy())
ax[0].fill_between(X_new.ravel(), y_new.ravel() -
y_std2, y_new.ravel()+y_std2, alpha=0.5)
ax[2].plot(losses)
ax[1].plot(X_new, model.get_LS(X_new, 0))
fig.savefig('./test_step_function.pdf')
def UFGPool(x, batch, batch_size, d_list, d_index, aggre_mode='sum'):
"""
Using Undecimated Framelet Transform for graph pooling.
:param x: batched hidden representation. shape: [# Node_Sum_Batch, # Hidden Units]
:param batch: batch index. shape: [# Node_Sum_Batch]
:param batch_size: integer batch size.
:param d_list: a list of matrix operators, where each element is a torch sparse tensor stored in a list.
:param d_index: a list of index tensors, where each element is a torch dense tensor used for aggregation.
:param aggre_mode: aggregation mode. choices: sum, max, and avg. (default: sum)
:return: batched vectorial representation for the graphs in the batch.
"""
if aggre_mode == 'sum':
f = global_add_pool
elif aggre_mode == 'avg':
f = global_mean_pool
elif aggre_mode == 'max':
f = global_max_pool
else:
raise Exception('aggregation mode is invalid')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
for i in range(batch_size):
# extract the i-th graph
bi = (batch == i)
coefs = torch.sparse.mm(scipy_to_torch_sparse(d_list[i][0]).to(device), x[bi, :])###shape 90 x 64
idx = torch.tensor(d_index[i][0]).to(device)
if i == 0:
x_pool = f(coefs, idx).flatten()#####output should be r-1 * lev+1 * 64
else:
x_pool = torch.vstack((x_pool, f(coefs, idx).flatten())) ##d_index contain all batch
return x_pool ##batch_size*64 vector
def padding_collate(batch: List[Tuple[torch.LongTensor, torch.Tensor, torch.Tensor]]) \
-> Dict[str, torch.Tensor]:
"""
Collate function generates tensors with a batch of data.
Args:
batch: Batch of data.
Returns:
input_ids: batch_size, pad_len
attention_mask: batch_size, pad_len
predicate_hot: batch_size, num_predicates
position_hot: batch_size, pad_len, num_predicates, 4
"""
input_ids, predicate_hot, position_hot = zip(*batch)
attention_mask = list(map(torch.ones_like, input_ids))
return {
'input_ids':
pad_sequence(input_ids, batch_first=True, padding_value=0),
'attention_mask':
pad_sequence(attention_mask, batch_first=True, padding_value=0),
'predicate_hot':
torch.vstack(predicate_hot),
'position_hot':
pad_sequence(position_hot, batch_first=True, padding_value=0.)
}
