def train_batch(param):
if len(memory) < param['batch_size']:
return 0
batch = memory.sample(param['batch_size'])
batch_states = default_states_preprocessor([m.state for m in batch])
batch_next_states = default_states_preprocessor([m.next_state for m in batch])
batch_ended = torch.tensor([m.ended for m in batch])
batch_rewards = torch.tensor([m.reward for m in batch]).to(device)
batch_actions = torch.tensor([m.action for m in batch]).to(device)
## Calculate expected reward:
with torch.set_grad_enabled(False):
not_ended_batch = 1 -torch.ByteTensor(batch_ended).to(device)
next_states_non_final = batch_next_states[not_ended_batch]
next_state_values = torch.zeros(param['batch_size']).to(device)
reward_hat = target_dqn(next_states_non_final)
next_state_values[not_ended_batch] = reward_hat.max(1)[0]
expected_state_action_values = next_state_values*param['GAMMA'] + batch_rewards
# Predict value function:
yhat = dqn(batch_states)
state_action_values = yhat.gather(1, batch_actions.unsqueeze(1)).squeeze()
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values)
optimizer.zero_grad()
loss.backward()
for param in dqn.parameters():
param.data.clamp_(-1, 1)
optimizer.step()
return float(loss.data.cpu().numpy())
def test_manual_bounds(self, cuda=False):
device = torch.device("cuda") if cuda else torch.device("cpu")
for dtype in (torch.float, torch.double):
# get a test module
train_x = torch.tensor([[1.0, 2.0, 3.0]], device=device, dtype=dtype)
train_y = torch.tensor([4.0], device=device, dtype=dtype)
likelihood = GaussianLikelihood()
model = ExactGP(train_x, train_y, likelihood)
model.covar_module = RBFKernel(ard_num_dims=3)
model.mean_module = ConstantMean()
model.to(device=device, dtype=dtype)
mll = ExactMarginalLogLikelihood(likelihood, model)
# test the basic case
x, pdict, bounds = module_to_array(
module=mll, bounds={"model.covar_module.raw_lengthscale": (0.1, None)}
)
self.assertTrue(np.array_equal(x, np.zeros(5)))
expected_sizes = {
"likelihood.noise_covar.raw_noise": torch.Size([1]),
"model.covar_module.raw_lengthscale": torch.Size([1, 3]),
"model.mean_module.constant": torch.Size([1]),
}
self.assertEqual(set(pdict.keys()), set(expected_sizes.keys()))
for pname, val in pdict.items():
self.assertEqual(val.dtype, dtype)
self.assertEqual(val.shape, expected_sizes[pname])
self.assertEqual(val.device.type, device.type)
lower_exp = np.full_like(x, 0.1)
for p in ("likelihood.noise_covar.raw_noise", "model.mean_module.constant"):
lower_exp[_get_index(pdict, p)] = -np.inf
self.assertTrue(np.equal(bounds[0], lower_exp).all())
self.assertTrue(np.equal(bounds[1], np.full_like(x, np.inf)).all())
def test_factory(self):
default_size = torch.Size([1, 3])
size = torch.Size([3, 3])
for include_size in [True, False]:
for use_tensor_idx in [True, False]:
for use_tensor_val in [True, False]:
for use_cuda in ([False] if not torch.cuda.is_available() else [True, False]):
# have to include size with cuda sparse tensors
include_size = include_size or use_cuda
dtype = torch.float64
long_dtype = torch.int64
device = torch.device('cpu') if not use_cuda else torch.device(torch.cuda.device_count() - 1)
indices = torch.tensor(([0], [2]), dtype=long_dtype) if use_tensor_idx else ([0], [2])
values = torch.tensor([1.], dtype=dtype) if use_tensor_val else 1.
if include_size:
sparse_tensor = torch.sparse_coo_tensor(indices, values, size, dtype=dtype,
device=device, requires_grad=True)
else:
sparse_tensor = torch.sparse_coo_tensor(indices, values, dtype=dtype,
device=device, requires_grad=True)
self.assertEqual(indices, sparse_tensor._indices())
self.assertEqual(values, sparse_tensor._values())
self.assertEqual(size if include_size else default_size, sparse_tensor.size())
self.assertEqual(dtype, sparse_tensor.dtype)
if use_cuda:
self.assertEqual(device, sparse_tensor._values().device)
self.assertEqual(True, sparse_tensor.requires_grad)
def test_cases_cos():
a=torch.tensor([[1,2,3.0],[0,0,0.0]])
b=torch.tensor([[1,2,3.1],[-1,-2,-3.0]])
a2=torch.tensor([[1,2,3.0],[0,0,0]])
b2=torch.tensor([[1,2,3.1],[-1,-2,-3.0],[5,5,5.0],[6,6,6.0]])
a3=torch.tensor([1,2,3.0])
b3=torch.tensor([1,2,3.1])
a31=torch.tensor([[1,2,3.0]])
b31=torch.tensor([[1,2,3.1]])
ar=np.random.rand(5,10)
br=np.random.rand(15,10)
art=torch.tensor(ar)
brt=torch.tensor(br)
cos(a,b)
cos(a2,b2)
abrt = cos(art,brt)
print("sklearn cos:", sklearn.metrics.pairwise.cosine_similarity(ar,br))
cos(a3,b3)
try:
cos(a3,b3)
except:
print("cos(a3,b3) failed")
try:
cos([a3],[b3])
except:
print("cos(a3,b3) failed")
cos(a31,b31)
def test_index_setitem_bools_slices(self):
true = torch.tensor(1, dtype=torch.uint8)
false = torch.tensor(0, dtype=torch.uint8)
tensors = [Variable(torch.randn(2, 3)), torch.tensor(3)]
for a in tensors:
# prefix with a 1,1, to ensure we are compatible with numpy which cuts off prefix 1s
# (some of these ops already prefix a 1 to the size)
neg_ones = torch.ones_like(a) * -1
neg_ones_expanded = neg_ones.unsqueeze(0).unsqueeze(0)
a[True] = neg_ones_expanded
self.assertEqual(a, neg_ones)
a[False] = 5
self.assertEqual(a, neg_ones)
a[true] = neg_ones_expanded * 2
self.assertEqual(a, neg_ones * 2)
a[false] = 5
self.assertEqual(a, neg_ones * 2)
a[None] = neg_ones_expanded * 3
self.assertEqual(a, neg_ones * 3)
a[...] = neg_ones_expanded * 4
self.assertEqual(a, neg_ones * 4)
if a.dim() == 0:
with self.assertRaises(RuntimeError):
a[:] = neg_ones_expanded * 5
def forward(self, sents, sent_lengths):
'''
sents is (batch_size by padded_length)
when we evaluate sentence by sentence, you evaluate it with batch_size = 1, padded_length.
[[1, 2, 3, 4]] etc.
'''
batch_size = sents.size()[0]
sent_lengths = list(sent_lengths)
# We sort and then do pad packed sequence here.
descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]
descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)]
descending_lengths = torch.tensor(descending_lengths)
descending_indices = torch.tensor(descending_indices).to(device)
descending_sents = torch.index_select(sents, torch.tensor(0), descending_indices)
# get embedding
embed = self.embedding(descending_sents)
# pack padded sequence
embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True)
# fprop though RNN
self.hidden = self.init_hidden(batch_size)
rnn_out, self.hidden = self.gru(embed, self.hidden)
pdb.set_trace()
rnn_out, _ = torch.nn.utils.rnn.pad_packed_sequence(rnn_out, batch_first=True)
# rnn_out is 32 by 72 by 256
# change the order back
change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))]
self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device))
rnn_out = torch.index_select(rnn_out, 0, torch.LongTensor(change_it_back).to(device))
return rnn_out, self.hidden
def generate_translation(encoder, decoder, sentence, max_length, target_lang, search="greedy", k = None):
"""
@param max_length: the max # of words that the decoder can return
@returns decoded_words: a list of words in target language
"""
with torch.no_grad():
input_tensor = sentence
input_length = sentence.size()[1]
# encode the source sentence
encoder_hidden = encoder.init_hidden(1)
# input_tensor 1 by 12
#
encoder_outputs, encoder_hidden = encoder(input_tensor.view(1, -1),torch.tensor([input_length]))
# start decoding
decoder_input = torch.tensor([[SOS_token]], device=device) # SOS
decoder_hidden = encoder_hidden
decoded_words = []
if search == 'greedy':
decoded_words = greedy_search_batch(decoder, decoder_input, encoder_outputs, decoder_hidden, max_length)
elif search == 'beam':
if k == None:
k = 2 # since k = 2 preforms badly
decoded_words = beam_search(decoder, decoder_input, encoder_outputs, decoder_hidden, max_length, k, target_lang)
return decoded_words
def __init__(self, mean, std):
super(Normalization, self).__init__()
# .view the mean and std to make them [C x 1 x 1] so that they can
# directly work with image Tensor of shape [B x C x H x W].
# B is batch size. C is number of channels. H is height and W is width.
self.mean = torch.tensor(mean).view(-1, 1, 1)
self.std = torch.tensor(std).view(-1, 1, 1)
def perform_val(multi_gpu, device, embedding_size, batch_size, backbone, carray, issame, nrof_folds = 10, tta = True):
if multi_gpu:
backbone = backbone.module # unpackage model from DataParallel
backbone = backbone.to(device)
else:
backbone = backbone.to(device)
backbone.eval() # switch to evaluation mode
idx = 0
embeddings = np.zeros([len(carray), embedding_size])
with torch.no_grad():
while idx + batch_size <= len(carray):
batch = torch.tensor(carray[idx:idx + batch_size][:, [2, 1, 0], :, :])
if tta:
fliped = hflip_batch(batch)
emb_batch = backbone(batch.to(device)).cpu() + backbone(fliped.to(device)).cpu()
embeddings[idx:idx + batch_size] = l2_norm(emb_batch)
else:
embeddings[idx:idx + batch_size] = backbone(batch.to(device)).cpu()
idx += batch_size
if idx < len(carray):
batch = torch.tensor(carray[idx:])
if tta:
fliped = hflip_batch(batch)
emb_batch = backbone(batch.to(device)).cpu() + backbone(fliped.to(device)).cpu()
embeddings[idx:] = l2_norm(emb_batch)
else:
embeddings[idx:] = backbone(batch.to(device)).cpu()
tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
roc_curve_tensor = transforms.ToTensor()(roc_curve)
return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def calc_loss(batch, net, tgt_net, gamma, device="cpu", save_prefix=None):
states, actions, rewards, dones, next_states = common.unpack_batch(batch)
batch_size = len(batch)
states_v = torch.tensor(states).to(device)
actions_v = torch.tensor(actions).to(device)
next_states_v = torch.tensor(next_states).to(device)
# next state distribution
next_distr_v, next_qvals_v = tgt_net.both(next_states_v)
next_actions = next_qvals_v.max(1)[1].data.cpu().numpy()
next_distr = tgt_net.apply_softmax(next_distr_v).data.cpu().numpy()
next_best_distr = next_distr[range(batch_size), next_actions]
dones = dones.astype(np.bool)
# project our distribution using Bellman update
proj_distr = common.distr_projection(next_best_distr, rewards, dones, Vmin, Vmax, N_ATOMS, gamma)
# calculate net output
distr_v = net(states_v)
state_action_values = distr_v[range(batch_size), actions_v.data]
state_log_sm_v = F.log_softmax(state_action_values, dim=1)
proj_distr_v = torch.tensor(proj_distr).to(device)
if save_prefix is not None:
pred = F.softmax(state_action_values, dim=1).data.cpu().numpy()
save_transition_images(batch_size, pred, proj_distr, next_best_distr, dones, rewards, save_prefix)
loss_v = -state_log_sm_v * proj_distr_v
return loss_v.sum(dim=1).mean()
def load(self, fdata, use_char=False, n_context=1, max_len=10):
sentences = self.preprocess(fdata)
x, y, char_x, lens = [], [], [], []
for wordseq, tagseq in sentences:
wiseq = [self.wdict.get(w, self.unk_wi) for w in wordseq]
tiseq = [self.tdict[t] for t in tagseq]
# 获取每个词汇的上下文
if n_context > 1:
x.append(self.get_context(wiseq, n_context))
else:
x.append(torch.tensor(wiseq, dtype=torch.long))
y.append(torch.tensor(tiseq, dtype=torch.long))
# 不足最大长度的部分用0填充
char_x.append(torch.tensor([
[self.cdict.get(c, self.unk_ci)
for c in w[:max_len]] + [0] * (max_len - len(w))
for w in wordseq
]))
lens.append(len(tiseq))
x = pad_sequence(x, True)
y = pad_sequence(y, True)
char_x = pad_sequence(char_x, True)
lens = torch.tensor(lens)
if use_char:
dataset = TensorDataset(x, y, char_x, lens)
else:
dataset = TensorDataset(x, y, lens)
return dataset
def test_advance_with_all_repeats_gets_blocked(self):
# all beams repeat (beam >= 1 repeat dummy scores)
beam_sz = 5
n_words = 100
repeat_idx = 47
ngram_repeat = 3
beam = Beam(beam_sz, 0, 1, 2, n_best=2,
exclusion_tokens=set(),
global_scorer=GlobalScorerStub(),
block_ngram_repeat=ngram_repeat)
for i in range(ngram_repeat + 4):
# predict repeat_idx over and over again
word_probs = torch.full((beam_sz, n_words), -float('inf'))
word_probs[0, repeat_idx] = 0
attns = torch.randn(beam_sz)
beam.advance(word_probs, attns)
if i <= ngram_repeat:
self.assertTrue(
beam.scores.equal(
torch.tensor(
[0] + [-float('inf')] * (beam_sz - 1))))
else:
self.assertTrue(
beam.scores.equal(torch.tensor(
[self.BLOCKED_SCORE] * beam_sz)))
def test_gather_extended_gold_tokens(self):
vocab_size = self.model._target_vocab_size
end_index = self.model._end_index
pad_index = self.model._pad_index
oov_index = self.model._oov_index
tok_index = 6 # some other arbitrary token
assert tok_index not in [end_index, pad_index, oov_index]
# first sentence tokens:
# 1: oov but not copied
# 2: not oov and not copied
# 3: not copied
# 4: not copied
# second sentence tokens:
# 1: not oov and copied
# 2: oov and copied
# 3: not copied
# 4: not copied
# shape: (batch_size, target_sequence_length)
target_tokens = torch.tensor([[oov_index, tok_index, end_index, pad_index],
[tok_index, oov_index, tok_index, end_index]])
# shape: (batch_size, trimmed_source_length)
source_token_ids = torch.tensor([[0, 1, 2, 3],
[0, 1, 0, 2]])
# shape: (batch_size, target_sequence_length)
target_token_ids = torch.tensor([[4, 5, 6, 7],
[1, 0, 3, 4]])
# shape: (batch_size, target_sequence_length)
result = self.model._gather_extended_gold_tokens(target_tokens, source_token_ids, target_token_ids)
# shape: (batch_size, target_sequence_length)
check = np.array([[oov_index, tok_index, end_index, pad_index],
[tok_index, vocab_size, tok_index, end_index]])
np.testing.assert_array_equal(result.numpy(), check)
def guide(num_particles):
q1 = pyro.param("q1", torch.tensor(pi1, requires_grad=True))
q2 = pyro.param("q2", torch.tensor(pi2, requires_grad=True))
with pyro.iarange("particles", num_particles):
z = pyro.sample("z", dist.Normal(q2, 1.0).expand_by([num_particles]))
zz = torch.exp(z) / (1.0 + torch.exp(z))
pyro.sample("y", dist.Bernoulli(q1 * zz))
def testBinaryEntropy(self):
p = torch.tensor([0.1, 0.02, 0.99, 0.5, 0.75, 0.8, 0.3, 0.4, 0.0, 1.0])
entropy, entropySum = binaryEntropy(p)
self.assertAlmostEqual(entropySum, 5.076676985, places=4)
self.assertAlmostEqual(entropySum, entropy.sum(), places=4)
self.assertAlmostEqual(entropy[0], 0.468995594, places=4)
self.assertAlmostEqual(entropy[1], 0.141440543, places=4)
self.assertAlmostEqual(entropy[2], 0.080793136, places=4)
self.assertEqual(entropy[8], 0.0)
self.assertEqual(entropy[9], 0.0)
p = torch.tensor([0.25, 0.25, 0.25, 0.25])
entropy, entropySum = binaryEntropy(p)
self.assertAlmostEqual(entropySum, 3.245112498, places=4)
self.assertAlmostEqual(entropySum, entropy.sum(), places=4)
p = torch.tensor([0.5, 0.5, 0.5, 0.5])
entropy, entropySum = binaryEntropy(p)
self.assertAlmostEqual(entropySum, 4.0, places=4)
self.assertAlmostEqual(entropySum, entropy.sum(), places=4)
self.assertAlmostEqual(entropy[0], 1.0, places=4)
self.assertAlmostEqual(entropy[1], 1.0, places=4)
self.assertAlmostEqual(entropy[2], 1.0, places=4)
self.assertAlmostEqual(entropy[3], 1.0, places=4)
def guide():
q1 = pyro.param("q1", torch.tensor(pi1, requires_grad=True))
q2 = pyro.param("q2", torch.tensor(pi2, requires_grad=True))
with pyro.iarange("particles", num_particles):
y = pyro.sample("y", dist.Bernoulli(q1).expand_by([num_particles]), infer={"enumerate": enumerate1})
if include_z:
pyro.sample("z", dist.Normal(q2 * y + 0.10, 1.0))
def model(num_particles):
with pyro.iarange("particles", num_particles):
q3 = pyro.param("q3", torch.tensor(pi3, requires_grad=True))
q4 = pyro.param("q4", torch.tensor(0.5 * (pi1 + pi2), requires_grad=True))
z = pyro.sample("z", dist.Normal(q3, 1.0).expand_by([num_particles]))
zz = torch.exp(z) / (1.0 + torch.exp(z))
pyro.sample("y", dist.Bernoulli(q4 * zz))
def test_advance_with_all_repeats_gets_blocked(self):
# all beams repeat (beam >= 1 repeat dummy scores)
beam_sz = 5
n_words = 100
repeat_idx = 47
ngram_repeat = 3
for batch_sz in [1, 3]:
beam = BeamSearch(
beam_sz, batch_sz, 0, 1, 2, 2,
torch.device("cpu"), GlobalScorerStub(), 0, 30,
False, ngram_repeat, set(),
torch.randint(0, 30, (batch_sz,)), False, 0.)
for i in range(ngram_repeat + 4):
# predict repeat_idx over and over again
word_probs = torch.full(
(batch_sz * beam_sz, n_words), -float('inf'))
word_probs[0::beam_sz, repeat_idx] = 0
attns = torch.randn(1, batch_sz * beam_sz, 53)
beam.advance(word_probs, attns)
if i <= ngram_repeat:
expected_scores = torch.tensor(
[0] + [-float('inf')] * (beam_sz - 1))\
.repeat(batch_sz, 1)
self.assertTrue(beam.topk_log_probs.equal(expected_scores))
else:
self.assertTrue(
beam.topk_log_probs.equal(
torch.tensor(self.BLOCKED_SCORE)
.repeat(batch_sz, beam_sz)))
def diamond_guide(dim):
p0 = torch.tensor(math.exp(-0.70), requires_grad=True)
p1 = torch.tensor(math.exp(-0.43), requires_grad=True)
pyro.sample("a1", dist.Bernoulli(p0))
for i in pyro.irange("irange", dim):
pyro.sample("b{}".format(i), dist.Bernoulli(p1))
pyro.sample("c1", dist.Bernoulli(p0))
def test_optimizers(factory):
optim = factory()
def model(loc, cov):
x = pyro.param("x", torch.randn(2))
y = pyro.param("y", torch.randn(3, 2))
z = pyro.param("z", torch.randn(4, 2).abs(), constraint=constraints.greater_than(-1))
pyro.sample("obs_x", dist.MultivariateNormal(loc, cov), obs=x)
with pyro.iarange("y_iarange", 3):
pyro.sample("obs_y", dist.MultivariateNormal(loc, cov), obs=y)
with pyro.iarange("z_iarange", 4):
pyro.sample("obs_z", dist.MultivariateNormal(loc, cov), obs=z)
loc = torch.tensor([-0.5, 0.5])
cov = torch.tensor([[1.0, 0.09], [0.09, 0.1]])
for step in range(100):
tr = poutine.trace(model).get_trace(loc, cov)
loss = -tr.log_prob_sum()
params = {name: pyro.param(name).unconstrained() for name in ["x", "y", "z"]}
optim.step(loss, params)
for name in ["x", "y", "z"]:
actual = pyro.param(name)
expected = loc.expand(actual.shape)
assert_equal(actual, expected, prec=1e-2,
msg='{} in correct: {} vs {}'.format(name, actual, expected))
def run_episode(self, episode, steps_accumulated=0):
start_time = time.time()
observation = self.env.reset()
state = torch.from_numpy(observation).to(self.config.device, dtype=torch.float32).unsqueeze(0)
for step in range(MAX_STEPS):
action = self.agent.get_action(state, step + steps_accumulated)
observation_next, _, done, _ = self.env.step(action.item())
if done:
state_next = None
self.total_step = np.hstack((self.total_step[1:], step + 1))
if self.is_success_episode(step):
reward = torch.tensor([1.0], dtype=torch.float32, device=self.config.device)
else:
reward = torch.tensor([-1.0], dtype=torch.float32, device=self.config.device)
else:
reward = torch.tensor([0.0], dtype=torch.float32, device=self.config.device)
state_next = torch.from_numpy(observation_next).to(self.config.device, dtype=torch.float32).unsqueeze(0)
self.agent.observe(state, action, state_next, reward)
if step % self.config.replay_interval == 0:
self.agent.learn(episode)
state = state_next
if done:
elapsed_time = round(time.time() - start_time, 3)
print('episode: {0}, steps: {1}, mean steps {2}, time: {3}'.format(episode, step, self.total_step.mean(), elapsed_time))
return step + 1
return MAX_STEPS
def model():
p2 = torch.tensor(torch.ones(2) / 2)
p3 = torch.tensor(torch.ones(3) / 3)
x2 = pyro.sample("x2", dist.OneHotCategorical(p2))
x3 = pyro.sample("x3", dist.OneHotCategorical(p3))
assert x2.shape == torch.Size([2]) + iarange_shape + p2.shape
assert x3.shape == torch.Size([3, 1]) + iarange_shape + p3.shape
def model():
with pyro.iarange("num_particles", 10, dim=-3):
with pyro.iarange("components", 2, dim=-1):
p = pyro.sample("p", dist.Beta(torch.tensor(1.1), torch.tensor(1.1)))
assert p.shape == torch.Size((10, 1, 2))
with pyro.iarange("data", data.shape[0], dim=-2):
pyro.sample("obs", dist.Bernoulli(p), obs=data)
def testDutyCycleUpdate(self):
"""
Start with equal duty cycle, boost factor=0, k=4, batch size=2
"""
x = self.x2
expected = torch.zeros_like(x)
expected[0, 0, 1, 0] = 1.1
expected[0, 0, 1, 1] = 1.2
expected[0, 1, 0, 1] = 1.2
expected[0, 2, 1, 0] = 1.3
expected[1, 0, 0, 0] = 1.4
expected[1, 1, 0, 0] = 1.5
expected[1, 1, 0, 1] = 1.6
expected[1, 2, 1, 1] = 1.7
dutyCycle = torch.zeros((1, 3, 1, 1))
dutyCycle[:] = 1.0 / 3.0
updateDutyCycleCNN(expected, dutyCycle, 2, 2)
newDuty = torch.tensor([1.5000, 1.5000, 1.0000]) / 4.0
diff = (dutyCycle.reshape(-1) - newDuty).abs().sum()
self.assertLessEqual(diff, 0.001)
dutyCycle[:] = 1.0 / 3.0
updateDutyCycleCNN(expected, dutyCycle, 4, 4)
newDuty = torch.tensor([0.3541667, 0.3541667, 0.2916667])
diff = (dutyCycle.reshape(-1) - newDuty).abs().sum()
self.assertLessEqual(diff, 0.001)
def postprocess_sequence(self, X):
"""Embed (variable-length) sequences
Parameters
----------
X : list
List of input sequences
Returns
-------
fX : numpy array
Batch of sequence embeddings.
"""
lengths = torch.tensor([len(x) for x in X])
sorted_lengths, sort = torch.sort(lengths, descending=True)
_, unsort = torch.sort(sort)
sequences = [torch.tensor(X[i],
dtype=torch.float32,
device=self.device) for i in sort]
padded = pad_sequence(sequences, batch_first=True, padding_value=0)
packed = pack_padded_sequence(padded, sorted_lengths,
batch_first=True)
cpu = torch.device('cpu')
fX = self.model(packed).detach().to(cpu).numpy()
return fX[unsort]
def predict_spec_sliding_window(spectrogram, model, chunk_size=256, jump=128, hierarchical_model=None, hierarchy_threshold=15):
"""
Generate the prediction sequence for a full audio sequence
using a sliding window. Slide the window by one spectrogram frame
and pass each window through the given model. Compute the average
over overlapping window predictions to get the final prediction.
Allow for having a hierarchical model! If using a hierarchical model
also save the model_0 predictions!
Return:
With Hierarchical Model - hierarchical predictions, model_0 predictions
Solo Model - predictions
"""
# Get the number of frames in the full audio clip
predictions = np.zeros(spectrogram.shape[0])
if hierarchical_model is not None:
hierarchical_predictions = np.zeros(spectrogram.shape[0])
# Keeps track of the number of predictions made for a given
# slice for final averaging!
overlap_counts = np.zeros(spectrogram.shape[0])
# This is a bit janky but we will manually transform
# each spectrogram chunk
#spectrogram = torch.from_numpy(spectrogram).float()
# Add a batch dim for the model!
#spectrogram = torch.unsqueeze(spectrogram, 0) # Shape - (1, time, freq)
# Added!
spectrogram = np.expand_dims(spectrogram,axis=0)
# For the sliding window we slide the window by one spectrogram
# frame, determined by the hop size.
spect_idx = 0 # The frame idx of the beginning of the current window
i = 0
# How can I parralelize this shit??????
while spect_idx + chunk_size <= spectrogram.shape[1]:
spect_slice = spectrogram[:, spect_idx: spect_idx + chunk_size, :]
# Transform the slice - this is definitely sketchy!!!!
spect_slice = (spect_slice - np.mean(spect_slice)) / np.std(spect_slice)
spect_slice = torch.from_numpy(spect_slice).float()
spect_slice = spect_slice.to(parameters.device)
outputs = model(spect_slice) # Shape - (1, chunk_size, 1)
compressed_out = outputs.view(-1, 1).squeeze()
# Now check if we are running the hierarchical model
if hierarchical_model is not None:
chunk_preds = torch.sigmoid(compressed_out)
binary_preds = torch.where(chunk_preds > parameters.THRESHOLD, torch.tensor(1.0).to(parameters.device), torch.tensor(0.0).to(parameters.device))
pred_counts = torch.sum(binary_preds)
# Check if we need to run the second model
hierarchical_compressed_out = compressed_out
if pred_counts.item() >= hierarchy_threshold:
hierarchical_outputs = hierarchical_model(spect_slice)
hierarchical_compressed_out = hierarchical_outputs.view(-1, 1).squeeze()
# Save the hierarchical model's output
hierarchical_predictions[spect_idx: spect_idx + chunk_size] += hierarchical_compressed_out.cpu().detach().numpy()
overlap_counts[spect_idx: spect_idx + chunk_size] += 1
# Save the model_0's output!
predictions[spect_idx: spect_idx + chunk_size] += compressed_out.cpu().detach().numpy()
spect_idx += jump
i += 1
# Do the last one if it was not covered
if (spect_idx - jump + chunk_size != spectrogram.shape[1]):
#print ('One final chunk!')
spect_slice = spectrogram[:, spect_idx: , :]
# Transform the slice
# Should use the function from the dataset!!
spect_slice = (spect_slice - np.mean(spect_slice)) / np.std(spect_slice)
spect_slice = torch.from_numpy(spect_slice).float()
spect_slice = spect_slice.to(parameters.device)
outputs = model(spect_slice) # Shape - (1, chunk_size, 1)
# In the case of ResNet the output is forced to the chunk size
compressed_out = outputs.view(-1, 1).squeeze()[:predictions[spect_idx: ].shape[0]]
# Now check if we are running the hierarchical model
if hierarchical_model is not None:
chunk_preds = torch.sigmoid(compressed_out)
binary_preds = torch.where(chunk_preds > parameters.THRESHOLD, torch.tensor(1.0).to(parameters.device), torch.tensor(0.0).to(parameters.device))
pred_counts = torch.sum(binary_preds)
# Check if we need to run the second model
hierarchical_compressed_out = compressed_out
if pred_counts.item() >= hierarchy_threshold:
hierarchical_outputs = hierarchical_model(spect_slice)
hierarchical_compressed_out = hierarchical_outputs.view(-1, 1).squeeze()[:predictions[spect_idx: ].shape[0]]
# Save the hierarchical model's output
hierarchical_predictions[spect_idx: ] += hierarchical_compressed_out.cpu().detach().numpy()
overlap_counts[spect_idx: ] += 1
# Save the model_0's output!
predictions[spect_idx: ] += compressed_out.cpu().detach().numpy()
# Average the predictions on overlapping frames
predictions = predictions / overlap_counts
if hierarchical_model is not None:
hierarchical_predictions = hierarchical_predictions / overlap_counts
# Get squashed [0, 1] predictions
predictions = sigmoid(predictions)
if hierarchical_model is not None:
hierarchical_predictions = sigmoid(hierarchical_predictions)
return hierarchical_predictions, predictions
return predictions
def test_simple_relu_conductance(self):
net = BasicModel_MultiLayer()
inp = torch.tensor([[0.0, 100.0, 0.0]])
self._conductance_test_assert(net, net.relu, inp,
[[90.0, 100.0, 100.0, 100.0]])
def test_simple_linear_conductance(self):
net = BasicModel_MultiLayer()
inp = torch.tensor([[0.0, 100.0, 0.0]], requires_grad=True)
self._conductance_test_assert(net, net.linear1, inp,
[[90.0, 100.0, 100.0, 100.0]])
def __getitem__(self, i):
return torch.tensor(self.examples[i], dtype=torch.long)
def avg(self):
d = torch.tensor(list(self.deque), dtype=torch.float32)
return d.mean().item()
def median(self): # @property 是装饰器，这里可简单理解为增加median属性(只读)
d = torch.tensor(list(self.deque))
return d.median().item()
def get_sample(self, idx):
assert type(idx) is int
data = self.__getitem__(idx)
return [torch.tensor(_dt.reshape(1, -1)) for _dt in data]
def generateTestCase (img, targetedNeuronIndex, desiredNAP, model):
""" Generate a test case to satisfy the specified neuronActivationPattern
Args:
img: an image to be perturbed towards the desired activation pattern
targetedNeuronIndex: Indices of neuron with the goal of satisfying the pattern
desiredNAP: desired shape (>0 is set to 1; <= 0 is set to -1)
model: neural network under analysis
"""
steps = 20000
loss = nn.L1Loss()
alpha = 0.1
x = Variable(img, requires_grad=True)
result = img
delta = torch.zeros(result.shape)
for step in range(steps):
if (step == steps -1):
print("Unable to find an image to satisfy the required pattern!")
zero_gradients(x)
out, intermediate = model.forwardWithIntermediate(x)
# The output label is first set to values equal to the computed layer.
labels = torch.tensor(intermediate)
# Modify the label, such that if the sign of the specified neuron is different, use the desired value. By doing so,
# (1) the loss of other un-specified neuron will be 0,
# (2) for specified neuron with correct sign, the loss will be 0
# (3) for specified neuron without correct sign, there will be a loss
satisfied = True
for i in range(len(targetedNeuronIndex)):
if not ((intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[i]]>0) == (desiredNAP[i]>0)):
satisfied = False
labels[0][targetedNeuronIndex[i]] = desiredNAP[i]
if satisfied:
if step == 0:
print("The original image already satisfy the required pattern!")
return result.cpu(), True
else :
print("Found an image to successfully create the required pattern:")
return result.cpu(), True
if(step == 0):
print(intermediate)
print(labels)
y = Variable(labels)
_loss = loss(intermediate, y)
_loss.backward()
result = x.data - (alpha * x.grad.data)
# TODO: Here we are not clamping the input
result = torch.clamp(result, 0.0, 1.0)
x.data = result
# x.data = x.data - (alpha * x.grad.data)
if(step % 500 == 0):
print(str(step)+": "+str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[0]]) + ", "+ str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[1]]))
return result.cpu(), False
def forward(self, x_acc, x_mic, acc_lens, mic_lens):
# (int(np.shape(x_acc)[0] / 5) is the current batch size
acc_s_batch = x_acc[0 : np.shape(x_acc)[0] : 5, :, :]
mic_s_batch = x_mic[0 : np.shape(x_mic)[0] : 5, :, :]
acc_s_lengths = torch.tensor(acc_lens[0 : np.shape(x_acc)[0] : 5])
mic_s_lengths = torch.tensor(mic_lens[0 : np.shape(x_acc)[0] : 5])
acc_s_batch = acc_s_batch.permute(0, 2, 1)
mic_s_batch = mic_s_batch.permute(0, 2, 1)
acc_s_batch = acc_s_batch.to(self.device)
mic_s_batch = mic_s_batch.to(self.device)
out_acc_s = self.tcn_subject_acc(acc_s_batch.float())
out_mic_s = self.tcn_subject_mic(mic_s_batch.float())
ending_acc_s_outputs = torch.zeros(
(int(np.shape(x_acc)[0] / 5), self.hidden_dim)
)
ending_mic_s_outputs = torch.zeros(
(int(np.shape(x_acc)[0] / 5), self.hidden_dim)
)
for b_num, cur_size in zip(np.arange(int(np.shape(x_acc)[0] / 5)), acc_s_lengths):
if cur_size == 0:
continue
else:
ending_acc_s_outputs[b_num, :] = out_acc_s[b_num, :, cur_size - 1]
for b_num, cur_size in zip(np.arange(int(np.shape(x_acc)[0] / 5)), mic_s_lengths):
if cur_size == 0:
continue
else:
ending_mic_s_outputs[b_num, :] = out_mic_s[b_num, :, cur_size - 1]
# Get last time step
out_acc_s_last_timestep = ending_acc_s_outputs.to(self.device)
out_mic_s_last_timestep = ending_mic_s_outputs.to(self.device)
out_neighbors_acc = []
out_neighbors_mic = []
# Neighbor's data
for i in np.arange(1, 5):
cur_acc_n_batch = x_acc[i : np.shape(x_acc)[0] : 5, :, :]
cur_mic_n_batch = x_mic[i : np.shape(x_mic)[0] : 5, :, :]
cur_acc_n_lengths = torch.tensor(acc_lens[i : np.shape(x_acc)[0] : 5])
cur_mic_n_lengths = torch.tensor(mic_lens[i : np.shape(x_mic)[0] : 5])
cur_acc_n_batch = cur_acc_n_batch.permute(0, 2, 1)
cur_mic_n_batch = cur_mic_n_batch.permute(0, 2, 1)
cur_acc_n_batch = cur_acc_n_batch.to(self.device)
cur_mic_n_batch = cur_mic_n_batch.to(self.device)
out_neighbor_acc = self.tcn_neighbors_acc(cur_acc_n_batch.float())
out_neighbor_mic = self.tcn_neighbors_mic(cur_mic_n_batch.float())
ending_acc_n_outputs = torch.zeros(
(int(np.shape(x_acc)[0] / 5), self.hidden_dim)
)
ending_mic_n_outputs = torch.zeros(
(int(np.shape(x_mic)[0] / 5), self.hidden_dim)
)
for b_num, cur_size in zip(
np.arange(int(np.shape(x_acc)[0] / 5)), cur_acc_n_lengths
):
if cur_size == 0:
continue
else:
ending_acc_n_outputs[b_num, :] = out_neighbor_acc[
b_num, :, cur_size - 1
]
for b_num, cur_size in zip(
np.arange(int(np.shape(x_acc)[0] / 5)), cur_mic_n_lengths
):
if cur_size == 0:
continue
else:
ending_mic_n_outputs[b_num, :] = out_neighbor_mic[
b_num, :, cur_size - 1
]
ending_acc_n_outputs = ending_acc_n_outputs.to(self.device)
ending_mic_n_outputs = ending_mic_n_outputs.to(self.device)
out_neighbors_acc.append(ending_acc_n_outputs)
out_neighbors_mic.append(ending_mic_n_outputs)
out_neighbors_acc.insert(0, out_acc_s_last_timestep)
out_neighbors_mic.insert(0, out_mic_s_last_timestep)
pooled_outputs_acc = []
pooled_outputs_mic = []
for bn in np.arange(int(np.shape(x_acc)[0] / 5)):
# Get the data of the subject entered the ESM
sbj_acc = out_neighbors_acc[0][bn, :].view(1, self.hidden_dim)
sbj_mic = out_neighbors_mic[0][bn, :].view(1, self.hidden_dim)
# Get the neighbours
n1_acc = out_neighbors_acc[1][bn, :].view(1, self.hidden_dim)
n2_acc = out_neighbors_acc[2][bn, :].view(1, self.hidden_dim)
n3_acc = out_neighbors_acc[3][bn, :].view(1, self.hidden_dim)
n4_acc = out_neighbors_acc[4][bn, :].view(1, self.hidden_dim)
n1_mic = out_neighbors_mic[1][bn, :].view(1, self.hidden_dim)
n2_mic = out_neighbors_mic[2][bn, :].view(1, self.hidden_dim)
n3_mic = out_neighbors_mic[3][bn, :].view(1, self.hidden_dim)
n4_mic = out_neighbors_mic[4][bn, :].view(1, self.hidden_dim)
if len(torch.nonzero(n1_acc)) != 0:
sbj_acc = torch.cat((sbj_acc, n1_acc), dim=0)
if len(torch.nonzero(n2_acc)) != 0:
sbj_acc = torch.cat((sbj_acc, n2_acc), dim=0)
if len(torch.nonzero(n3_acc)) != 0:
sbj_acc = torch.cat((sbj_acc, n3_acc), dim=0)
if len(torch.nonzero(n4_acc)) != 0:
sbj_acc = torch.cat((sbj_acc, n4_acc), dim=0)
if len(torch.nonzero(n1_mic)) != 0:
sbj_mic = torch.cat((sbj_mic, n1_mic), dim=0)
if len(torch.nonzero(n2_mic)) != 0:
sbj_mic = torch.cat((sbj_mic, n2_mic), dim=0)
if len(torch.nonzero(n3_mic)) != 0:
sbj_mic = torch.cat((sbj_mic, n3_mic), dim=0)
if len(torch.nonzero(n4_mic)) != 0:
sbj_mic = torch.cat((sbj_mic, n4_mic), dim=0)
avg_pooled_acc = torch.mean(sbj_acc, dim=0)
pooled_outputs_acc.append(avg_pooled_acc)
avg_pooled_mic = torch.mean(sbj_mic, dim=0)
pooled_outputs_mic.append(avg_pooled_mic)
pooled_outputs_acc = torch.stack(pooled_outputs_acc)
pooled_outputs_mic = torch.stack(pooled_outputs_mic)
all_out = torch.cat((pooled_outputs_acc, pooled_outputs_mic), 1)
all_out_eff = self.drp_eff(F.leaky_relu(self.fc_eff(all_out)))
all_out_fst = self.drp_fst(F.leaky_relu(self.fc_fst(all_out)))
all_out_sts = self.drp_sts(F.leaky_relu(self.fc_sts(all_out)))
out_eff = self.out_eff(all_out_eff)
out_fst = self.out_fst(all_out_fst)
out_sts = self.out_sts(all_out_sts)
return out_eff, out_fst, out_sts
def evaluate(args,
model: PreTrainedModel,
tokenizer: PreTrainedTokenizer,
prefix="") -> Dict:
# Loop to handle MNLI double evaluation (matched, mis-matched)
eval_output_dir = args.output_dir
eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True)
if args.local_rank in [-1, 0]:
os.makedirs(eval_output_dir, exist_ok=True)
args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
# Note that DistributedSampler samples randomly
def collate(examples: List[torch.Tensor]):
if tokenizer._pad_token is None:
return pad_sequence(examples, batch_first=True)
return pad_sequence(examples,
batch_first=True,
padding_value=tokenizer.pad_token_id)
eval_sampler = SequentialSampler(eval_dataset)
eval_dataloader = DataLoader(eval_dataset,
sampler=eval_sampler,
batch_size=args.eval_batch_size,
collate_fn=collate)
# multi-gpu evaluate
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Num examples = %d", len(eval_dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
eval_loss = 0.0
nb_eval_steps = 0
model.eval()
for batch in tqdm(eval_dataloader, desc="Evaluating"):
inputs, labels = mask_tokens(batch, tokenizer,
args) if args.mlm else (batch, batch)
inputs = inputs.to(args.device)
labels = labels.to(args.device)
with torch.no_grad():
outputs = model(inputs,
masked_lm_labels=labels) if args.mlm else model(
inputs, labels=labels)
lm_loss = outputs[0]
eval_loss += lm_loss.mean().item()
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
perplexity = torch.exp(torch.tensor(eval_loss))
result = {"perplexity": perplexity}
output_eval_file = os.path.join(eval_output_dir, prefix,
"eval_results.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results {} *****".format(prefix))
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
return result
def reference(ref_data):
torch_test_data = torch.tensor(input_data, requires_grad=False)
torch_softshrink = torch.nn.Softshrink(lambd=1.5)
m = torch_softshrink(torch_test_data)
return [m]
def visualize_output(
pdf: t.Tensor,
ex: batched_example.BatchedExample,
task_type: configuration.TaskType,
batch_indices: Iterable[int] = None) -> List[List[t.Tensor]]:
"""Visualizes the predicted outputs next to the ground-truth.
Args:
pdf: The class probabilities output by the model.
ex: The input batch, containing the ground-truth among others.
task_type: The task type.
batch_indices: The batch elements to visualized. Visualizes all by default.
Returns:
One list of images for each batch element, containing the element
visualized from different angles.
"""
palette = misc_util.to_tensor(colors.DEFAULT_COLOR_PALETTE, t.float32)
palette = palette.to(pdf.device)
output_images = []
scene_num_tri = [n.sum().item() for n in ex.mesh_num_tri]
offsets = t.tensor(scene_num_tri).cumsum(0).tolist()
offsets = [0] + offsets[:-1]
if not batch_indices:
batch_indices = range(pdf.shape[0])
vis = visualization_artifacts
pred_lbl, gt_lbl = extract_labels(pdf, ex, task_type)
for batch_idx in batch_indices:
v2x = ex.v2x_transform[batch_idx].inverse()
gt_mesh_labels = ex.mesh_labels[batch_idx]
artifacts_3d = []
# Marching cubes visualization for the predicted volume
if task_type == configuration.TaskType.FG_BG:
assert gt_mesh_labels.shape == (1, )
mc_colors = palette.new_tensor([0, gt_mesh_labels[0]],
dtype=t.int64)
mc_colors = palette[mc_colors]
else:
num_classes = pdf.shape[1]
mc_colors = palette[:num_classes]
artifacts_3d.append(
vis.MarchingCubesArtifact(pdf[batch_idx], v2x, mc_colors))
# Mesh visualization of the GT scene
gt_mesh_colors = palette[gt_mesh_labels.to(t.int64)]
mesh_num_tri = ex.mesh_num_tri[batch_idx]
offset = offsets[batch_idx]
gt_vertices = ex.vertices[offset:offset + scene_num_tri[batch_idx]]
artifacts_3d.append(
vis.MultiMeshArtifact(gt_vertices,
mesh_num_tri,
mesh_colors=gt_mesh_colors))
# Voxel grid visualizations of both the predicted volume and the GT
artifacts_3d.append(vis.VoxelGridArtifact(pred_lbl[batch_idx], v2x))
artifacts_3d.append(vis.VoxelGridArtifact(gt_lbl[batch_idx], v2x))
artifacts = [
vis.ImageArtifact(ex.input_image[batch_idx]), artifacts_3d
]
camera_images = vis.visualize_artifacts(artifacts,
ex.camera_transform[batch_idx],
ex.view_transform[batch_idx])
output_images.append(camera_images)
return output_images
def __init__(self):
super(Norm, self).__init__()
self.gamma = Parameter(torch.tensor(1.0))
self.beta = Parameter(torch.tensor(0.0))
def _calculate_per_channel_qparams(self, min_vals, max_vals):
# type: (Tensor, Tensor) -> Tuple[Tensor, Tensor]
r"""Calculates the per channel quantization parameters, given min and max
value tensors.
Args:
min_vals: Minimum values per channel
max_vals: Maximum values per channel
Returns:
scales: Per channel scales tensor of shape (#channels,)
zero_points: Per channel zero points tensor of shape (#channels,)
"""
if min_vals.numel() == 0 or max_vals.numel() == 0:
warnings.warn("must run observer before calling calculate_qparams.\
Returning default scale and zero point ")
return torch.tensor([1.0]), torch.tensor([0])
diff = min_vals <= max_vals
assert (torch.sum(diff) == len(diff)
), "min_vals should be less than max_vals for indices."
scales = torch.empty(min_vals.size(), dtype=torch.float32)
zero_points = torch.empty(min_vals.size(), dtype=torch.int64)
if self.dtype == torch.qint8:
if self.reduce_range:
qmin, qmax = -64, 63
else:
qmin, qmax = -128, 127
else:
if self.reduce_range:
qmin, qmax = 0, 127
else:
qmin, qmax = 0, 255
max_vals, min_vals = max_vals.to(dtype=torch.float), min_vals.to(
dtype=torch.float)
min_vals = torch.min(
min_vals,
torch.tensor([0.], device=min_vals.device, dtype=torch.float))
max_vals = torch.max(
max_vals,
torch.tensor([0.], device=max_vals.device, dtype=torch.float))
if torch.equal(max_vals, min_vals):
scales.fill_(1.0)
zero_points.fill_(0)
else:
if self.qscheme == torch.per_tensor_symmetric or self.qscheme == torch.per_channel_symmetric:
max_vals = torch.max(-min_vals, max_vals)
scales = max_vals / ((qmax - qmin) / 2)
scales = torch.max(
scales,
torch.tensor([self.eps],
device=scales.device,
dtype=scales.dtype))
if self.dtype == torch.qint8:
zp = 0
else:
zp = 128
zero_points.fill_(zp)
else:
scales = (max_vals - min_vals) / float(qmax - qmin)
scales = torch.max(
scales, torch.tensor([self.eps], device=scales.device))
zero_points = qmin - torch.round(min_vals / scales)
zero_points = torch.max(
zero_points,
torch.tensor([qmin],
dtype=zero_points.dtype,
device=zero_points.device))
zero_points = torch.min(
zero_points,
torch.tensor([qmax],
dtype=zero_points.dtype,
device=zero_points.device))
zero_points = zero_points.to(dtype=torch.int64)
scales.to(dtype=torch.float)
return scales, zero_points
def evaluate(args, model, processor, tokenizer, global_step, input_dir, prefix=""):
retrievers = dict()
for kg in args.use_kgs:
logger.info("Initialize kg:{}".format(kg))
kg_path = os.path.join(input_dir, args.kg_paths[kg])
data_path = os.path.join(args.data_dir, args.kg_paths[kg])
if not os.path.exists(kg_path):
logger.warning("need prepare training dataset firstly, program exit")
exit()
retrievers[kg] = initialize_kg_retriever(kg, kg_path, data_path, args.cache_file_suffix)
dataset, examples_tokenized, features, wn_synset_graphs, wn_synset_graphs_label_dict = \
load_and_cache_examples(args,
processor,
retrievers,
relation_list=args.relation_list,
input_dir=input_dir,
evaluate=True,
output_examples=True)
if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
os.mkdir(args.output_dir)
args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
# Note that DistributedSampler samples randomly
eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset, shuffle=False)
eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
# synset_graphs_batch = []
# for batch_index in eval_dataloader.batch_sampler:
# synset_graphs_batch.append([i for i in batch_index])
# multi-gpu evaluate
if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):
model = torch.nn.DataParallel(model)
if args.local_rank != -1 and not isinstance(model, torch.nn.parallel.DistributedDataParallel):
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True
)
# Eval!
logger.info("***** Running evaluation {} *****".format(prefix))
logger.info(" Dataset size = %d", len(dataset))
logger.info(" Batch size = %d", args.eval_batch_size)
if args.local_rank == -1:
logger.warning("program exits and please use pytorch DDP framework")
exit()
else:
# all_results = []
# all_start_logits = torch.tensor([], dtype=torch.float32, device=args.device)
# all_end_logits = torch.tensor([], dtype=torch.float32, device=args.device)
# all_unique_ids = []
all_pred = torch.tensor([], dtype=torch.long, device=args.device)
all_label_ids = torch.tensor([], dtype=torch.long, device=args.device)
all_question_ids = torch.tensor([], dtype=torch.long, device=args.device)
# start_time = timeit.default_timer()
epoch_iterator = tqdm(eval_dataloader, desc="Evaluating Iteration", disable=args.local_rank not in [-1, 0])
for step, batch in enumerate(epoch_iterator):
model.eval()
batch = tuple(t.to(args.device) for t in batch)
batch_synset_graphs = batch[3]
with torch.no_grad():
inputs = create_input(args, batch, global_step, batch_synset_graphs=batch_synset_graphs,
wn_synset_graphs=wn_synset_graphs, evaluate=True)
feature_indices = batch[3]
outputs = model(**inputs)
logits, label_ids, qas_ids = outputs[1], outputs[2], outputs[3]
all_pred = torch.cat((all_pred, torch.argmax(logits, axis=-1)), dim=0)
all_label_ids = torch.cat((all_label_ids, label_ids), dim=0)
all_question_ids = torch.cat((all_question_ids, qas_ids), dim=0)
start_time = timeit.default_timer()
all_pred_list = [torch.zeros_like(all_pred, device=args.device) for _ in
range(torch.distributed.get_world_size())]
all_label_ids_list = [torch.zeros_like(all_label_ids, device=args.device) for _ in
range(torch.distributed.get_world_size())]
all_question_ids_list = [torch.zeros_like(all_question_ids, device=args.device) for _ in
range(torch.distributed.get_world_size())]
all_gather(all_pred_list, all_pred)
all_gather(all_label_ids_list, all_label_ids)
all_gather(all_question_ids_list, all_question_ids)
logger.info(
"time for gather communication:{} in rank {}".format(timeit.default_timer() - start_time, args.local_rank))
if args.local_rank == 0:
start_time = timeit.default_timer()
all_results = []
all_pred_list = all_pred_list
all_label_ids_list = all_label_ids_list
all_question_ids_list = all_question_ids_list
preds = np.asarray([], dtype=int)
label_values = np.asarray([], dtype=int)
question_ids = np.asarray([], dtype=int)
for batch_idx, batch_preds in enumerate(all_pred_list):
preds = np.concatenate((preds, batch_preds.cpu().detach().numpy()), axis=0)
label_values = np.concatenate((label_values, all_label_ids_list[batch_idx].cpu().detach().numpy()),
axis=0)
question_ids = np.concatenate((question_ids, all_question_ids_list[batch_idx].cpu().detach().numpy()),
axis=0)
if not args.test:
acc = float((preds == label_values).mean())
results = {'acc': acc}
else:
results = None
if args.write_preds:
guids = []
for f in features:
guids.append(f.guid[0])
guids = np.asarray(guids, dtype='<U18')
assert len(preds)==len(guids)
write_prediction(preds, guids, "copa", prefix)
return results
else:
return None
def main():
parser = argparse.ArgumentParser()
## Required parameters
parser.add_argument("--bert_model", default=None, type=str, required=True,
help="Bert pre-trained model selected in the list: bert-base-uncased, "
"bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese.")
parser.add_argument("--output_dir", default=None, type=str, required=True,
help="The output directory where the model checkpoints will be written.")
## Other parameters
parser.add_argument("--train_file", default=None, type=str, help="SQuAD json for training. E.g., train-v1.1.json")
parser.add_argument("--predict_file", default=None, type=str,
help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json")
parser.add_argument("--max_seq_length", default=384, type=int,
help="The maximum total input sequence length after WordPiece tokenization. Sequences "
"longer than this will be truncated, and sequences shorter than this will be padded.")
parser.add_argument("--doc_stride", default=128, type=int,
help="When splitting up a long document into chunks, how much stride to take between chunks.")
parser.add_argument("--max_query_length", default=64, type=int,
help="The maximum number of tokens for the question. Questions longer than this will "
"be truncated to this length.")
parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.")
parser.add_argument("--do_predict", default=False, action='store_true', help="Whether to run eval on the dev set.")
parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.")
parser.add_argument("--predict_batch_size", default=8, type=int, help="Total batch size for predictions.")
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
parser.add_argument("--num_train_epochs", default=3.0, type=float,
help="Total number of training epochs to perform.")
parser.add_argument("--warmup_proportion", default=0.1, type=float,
help="Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% "
"of training.")
parser.add_argument("--n_best_size", default=20, type=int,
help="The total number of n-best predictions to generate in the nbest_predictions.json "
"output file.")
parser.add_argument("--max_answer_length", default=30, type=int,
help="The maximum length of an answer that can be generated. This is needed because the start "
"and end predictions are not conditioned on one another.")
parser.add_argument("--verbose_logging", default=False, action='store_true',
help="If true, all of the warnings related to data processing will be printed. "
"A number of warnings are expected for a normal SQuAD evaluation.")
parser.add_argument("--no_cuda",
default=False,
action='store_true',
help="Whether not to use CUDA when available")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
parser.add_argument('--gradient_accumulation_steps',
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.")
parser.add_argument("--local_rank",
type=int,
default=-1,
help="local_rank for distributed training on gpus")
parser.add_argument('--optimize_on_cpu',
default=False,
action='store_true',
help="Whether to perform optimization and keep the optimizer averages on CPU")
parser.add_argument('--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
parser.add_argument('--loss_scale',
type=float, default=128,
help='Loss scaling, positive power of 2 values can improve fp16 convergence.')
parser.add_argument('--do_lower_case', action="store_true", default=True, help="Lowercase the input")
args = parser.parse_args()
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
n_gpu = torch.cuda.device_count()
else:
device = torch.device("cuda", args.local_rank)
n_gpu = 1
# Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.distributed.init_process_group(backend='nccl')
if args.fp16:
logger.info("16-bits training currently not supported in distributed training")
args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496)
logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits trainiing: {}".format(
device, n_gpu, bool(args.local_rank != -1), args.fp16))
if args.gradient_accumulation_steps < 1:
raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
args.gradient_accumulation_steps))
args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if n_gpu > 0:
torch.cuda.manual_seed_all(args.seed)
if not args.do_train and not args.do_predict:
raise ValueError("At least one of `do_train` or `do_predict` must be True.")
if args.do_train:
if not args.train_file:
raise ValueError(
"If `do_train` is True, then `train_file` must be specified.")
if args.do_predict:
if not args.predict_file:
raise ValueError(
"If `do_predict` is True, then `predict_file` must be specified.")
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
raise ValueError("Output directory () already exists and is not empty.")
os.makedirs(args.output_dir, exist_ok=True)
tokenizer = BertTokenizer.from_pretrained(args.bert_model)
train_examples = None
num_train_steps = None
if args.do_train:
train_examples = read_squad_examples(
input_file=args.train_file, is_training=True)
num_train_steps = int(
len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
# Prepare model
model = BertForQuestionAnswering.from_pretrained(args.bert_model)
if args.fp16:
model.half()
model.to(device)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
elif n_gpu > 1:
model = torch.nn.DataParallel(model)
# Prepare optimizer
if args.fp16:
param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \
for n, param in model.named_parameters()]
elif args.optimize_on_cpu:
param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \
for n, param in model.named_parameters()]
else:
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}
]
optimizer = BertAdam(optimizer_grouped_parameters,
lr=args.learning_rate,
warmup=args.warmup_proportion,
t_total=num_train_steps)
global_step = 0
if args.do_train:
train_features = convert_examples_to_features(
examples=train_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=True)
logger.info("***** Running training *****")
logger.info(" Num orig examples = %d", len(train_examples))
logger.info(" Num split examples = %d", len(train_features))
logger.info(" Batch size = %d", args.train_batch_size)
logger.info(" Num steps = %d", num_train_steps)
all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long)
all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_start_positions, all_end_positions)
if args.local_rank == -1:
train_sampler = RandomSampler(train_data)
else:
train_sampler = DistributedSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)
model.train()
for _ in trange(int(args.num_train_epochs), desc="Epoch"):
ep = 0
for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
if n_gpu == 1:
batch = tuple(t.to(device) for t in batch) # multi-gpu does scattering it-self
input_ids, input_mask, segment_ids, start_positions, end_positions = batch
loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions)
if n_gpu > 1:
loss = loss.mean() # mean() to average on multi-gpu.
if args.fp16 and args.loss_scale != 1.0:
# rescale loss for fp16 training
# see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html
loss = loss * args.loss_scale
if args.gradient_accumulation_steps > 1:
loss = loss / args.gradient_accumulation_steps
loss.backward()
if (step + 1) % args.gradient_accumulation_steps == 0:
if args.fp16 or args.optimize_on_cpu:
if args.fp16 and args.loss_scale != 1.0:
# scale down gradients for fp16 training
for param in model.parameters():
if param.grad is not None:
param.grad.data = param.grad.data / args.loss_scale
is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True)
if is_nan:
logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling")
args.loss_scale = args.loss_scale / 2
model.zero_grad()
continue
optimizer.step()
copy_optimizer_params_to_model(model.named_parameters(), param_optimizer)
else:
optimizer.step()
model.zero_grad()
global_step += 1
torch.save(model.state_dict(), (args.output_dir+ "train_epoch" + ep + ".json"))
ep = ep +1
if args.do_predict:
eval_examples = read_squad_examples(
input_file=args.predict_file, is_training=False)
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
is_training=False)
logger.info("***** Running predictions *****")
logger.info(" Num orig examples = %d", len(eval_examples))
logger.info(" Num split examples = %d", len(eval_features))
logger.info(" Batch size = %d", args.predict_batch_size)
all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_example_index)
if args.local_rank == -1:
eval_sampler = SequentialSampler(eval_data)
else:
eval_sampler = DistributedSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.predict_batch_size)
model.eval()
all_results = []
logger.info("Start evaluating")
for input_ids, input_mask, segment_ids, example_indices in tqdm(eval_dataloader, desc="Evaluating"):
if len(all_results) % 1000 == 0:
logger.info("Processing example: %d" % (len(all_results)))
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
with torch.no_grad():
batch_start_logits, batch_end_logits = model(input_ids, segment_ids, input_mask)
for i, example_index in enumerate(example_indices):
start_logits = batch_start_logits[i].detach().cpu().tolist()
end_logits = batch_end_logits[i].detach().cpu().tolist()
eval_feature = eval_features[example_index.item()]
unique_id = int(eval_feature.unique_id)
all_results.append(RawResult(unique_id=unique_id,
start_logits=start_logits,
end_logits=end_logits))
output_prediction_file = os.path.join(args.output_dir, "predictions.json")
output_nbest_file = os.path.join(args.output_dir, "nbest_predictions.json")
write_predictions(eval_examples, eval_features, all_results,
args.n_best_size, args.max_answer_length,
args.do_lower_case, output_prediction_file,
output_nbest_file, args.verbose_logging)
def forward(self, im_data, im_info, gt_boxes, num_boxes, support_ims, all_cls_gt_boxes):
im_batch = im_data
im_info = im_info.data
gt_boxes = gt_boxes.data
num_boxes = num_boxes.data
support_ims = support_ims.data
batch_size = im_batch.size(0)
x = self.resnet_base(im_batch)
pos_sup_ims = support_ims[:, :self.num_shot, :, :, :].contiguous()
pos_sup_feats = self.resnet_base(pos_sup_ims.view(
batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80]
if self.training:
neg_sup_ims = support_ims[:, self.num_shot:, :, :, :].contiguous()
neg_sup_feats = self.resnet_base(neg_sup_ims.view(
batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80]
x1 = self.layer1(x)
x2 = self.layer2(x1)
x3 = self.layer3(x2)
x4 = self.layer4(x3)
pos_s1 = self.layer1(pos_sup_feats)
pos_s2 = self.layer2(pos_s1)
pos_s3 = self.layer3(pos_s2)
pos_s4 = self.layer4(pos_s3)
if self.training:
neg_s1 = self.layer1(neg_sup_feats)
neg_s2 = self.layer2(neg_s1)
neg_s3 = self.layer3(neg_s2)
neg_s4 = self.layer4(neg_s3)
features = self.fpn([x2, x3, x4])[1:]
pos_s_features = self.fpn([pos_s2, pos_s3, pos_s4])[1:]
if self.training:
neg_s_features = self.fpn([neg_s2, neg_s3, neg_s4])[1:]
correlation_features = self.attention_module(features, pos_s_features)
if self.training:
neg_correlation_features = self.attention_module(features, neg_s_features)
regression = torch.cat([self.regressionModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, 4]
classification = torch.cat([self.classificationModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, n_class]
if self.training:
neg_regression = torch.cat([self.regressionModel(feat) for feat in neg_correlation_features], dim=1)
neg_classification = torch.cat([self.classificationModel(feat) for feat in neg_correlation_features], dim=1)
anchors = self.anchors(im_batch)
rpn_loss_cls = torch.zeros(1).cuda()
rpn_loss_bbox = torch.zeros(1).cuda()
RCNN_loss_cls = torch.zeros(1).cuda()
RCNN_loss_bbox = torch.zeros(1).cuda()
rois_label = torch.zeros(10).cuda()
rois = None
cls_prob = None
bbox_pred = None
if self.training:
pos_RCNN_loss_cls, RCNN_loss_bbox = self.focalLoss(classification, regression, anchors, gt_boxes)
empty_gt_boxes = torch.zeros_like(gt_boxes) - 1.
neg_RCNN_loss_cls, _ = self.focalLoss(neg_classification, neg_regression, anchors, empty_gt_boxes)
RCNN_loss_cls = (pos_RCNN_loss_cls + neg_RCNN_loss_cls) / 2
return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label
else:
transformed_anchors = self.regressBoxes(anchors, regression)
transformed_anchors = self.clipBoxes(transformed_anchors, im_batch)
finalResult = [[], [], []]
finalScores = torch.Tensor([])
finalAnchorBoxesIndexes = torch.Tensor([]).long()
finalAnchorBoxesCoordinates = torch.Tensor([])
if torch.cuda.is_available():
finalScores = finalScores.cuda()
finalAnchorBoxesIndexes = finalAnchorBoxesIndexes.cuda()
finalAnchorBoxesCoordinates = finalAnchorBoxesCoordinates.cuda()
# scores = torch.squeeze(classification[:, :, 1])
scores = F.softmax(classification[0], 1)[:, 1]
scores_over_thresh = (scores > 0.5)
if scores_over_thresh.sum() == 0:
return None, None, np.array([0., 0., 0., 0.])
scores = scores[scores_over_thresh]
anchorBoxes = torch.squeeze(transformed_anchors)
anchorBoxes = anchorBoxes[scores_over_thresh]
anchors_nms_idx = nms(anchorBoxes, scores, 0.5)
finalScores = torch.cat((finalScores, scores[anchors_nms_idx])) # [n_predict]
finalAnchorBoxesIndexesValue = torch.tensor([1] * anchors_nms_idx.shape[0])
if torch.cuda.is_available():
finalAnchorBoxesIndexesValue = finalAnchorBoxesIndexesValue.cuda()
finalAnchorBoxesIndexes = torch.cat((finalAnchorBoxesIndexes, finalAnchorBoxesIndexesValue)) # [n_predict]
finalAnchorBoxesCoordinates = torch.cat((finalAnchorBoxesCoordinates, anchorBoxes[anchors_nms_idx])) # [n_predict, 4]
return [finalScores, finalAnchorBoxesIndexes, finalAnchorBoxesCoordinates]
def fit_single_frame(img,
keypoints,
body_model,
camera,
joint_weights,
body_pose_prior,
jaw_prior,
left_hand_prior,
right_hand_prior,
shape_prior,
expr_prior,
angle_prior,
result_fn='out.pkl',
mesh_fn='out.obj',
out_img_fn='overlay.png',
loss_type='smplify',
use_cuda=True,
init_joints_idxs=(9, 12, 2, 5),
use_face=True,
use_hands=True,
data_weights=None,
body_pose_prior_weights=None,
hand_pose_prior_weights=None,
jaw_pose_prior_weights=None,
shape_weights=None,
expr_weights=None,
hand_joints_weights=None,
face_joints_weights=None,
depth_loss_weight=1e2,
interpenetration=True,
coll_loss_weights=None,
df_cone_height=0.5,
penalize_outside=True,
max_collisions=8,
point2plane=False,
part_segm_fn='',
focal_length=5000.,
side_view_thsh=25.,
rho=100,
vposer_latent_dim=32,
vposer_ckpt='',
use_joints_conf=False,
interactive=True,
visualize=True,
save_meshes=True,
degrees=None,
batch_size=1,
dtype=torch.float32,
ign_part_pairs=None,
left_shoulder_idx=2,
right_shoulder_idx=5,
**kwargs):
assert batch_size == 1, 'PyTorch L-BFGS only supports batch_size == 1'
device = torch.device('cuda') if use_cuda else torch.device('cpu')
if degrees is None:
degrees = [0, 90, 180, 270]
if data_weights is None:
data_weights = [1, ] * 5
if body_pose_prior_weights is None:
body_pose_prior_weights = [4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78]
msg = (
'Number of Body pose prior weights {}'.format(
len(body_pose_prior_weights)) +
' does not match the number of data term weights {}'.format(
len(data_weights)))
assert (len(data_weights) ==
len(body_pose_prior_weights)), msg
if use_hands:
if hand_pose_prior_weights is None:
hand_pose_prior_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1]
msg = ('Number of Body pose prior weights does not match the' +
' number of hand pose prior weights')
assert (len(hand_pose_prior_weights) ==
len(body_pose_prior_weights)), msg
if hand_joints_weights is None:
hand_joints_weights = [0.0, 0.0, 0.0, 1.0]
msg = ('Number of Body pose prior weights does not match the' +
' number of hand joint distance weights')
assert (len(hand_joints_weights) ==
len(body_pose_prior_weights)), msg
if shape_weights is None:
shape_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1]
msg = ('Number of Body pose prior weights = {} does not match the' +
' number of Shape prior weights = {}')
assert (len(shape_weights) ==
len(body_pose_prior_weights)), msg.format(
len(shape_weights),
len(body_pose_prior_weights))
if use_face:
if jaw_pose_prior_weights is None:
jaw_pose_prior_weights = [[x] * 3 for x in shape_weights]
else:
jaw_pose_prior_weights = map(lambda x: map(float, x.split(',')),
jaw_pose_prior_weights)
jaw_pose_prior_weights = [list(w) for w in jaw_pose_prior_weights]
msg = ('Number of Body pose prior weights does not match the' +
' number of jaw pose prior weights')
assert (len(jaw_pose_prior_weights) ==
len(body_pose_prior_weights)), msg
if expr_weights is None:
expr_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1]
msg = ('Number of Body pose prior weights = {} does not match the' +
' number of Expression prior weights = {}')
assert (len(expr_weights) ==
len(body_pose_prior_weights)), msg.format(
len(body_pose_prior_weights),
len(expr_weights))
if face_joints_weights is None:
face_joints_weights = [0.0, 0.0, 0.0, 1.0]
msg = ('Number of Body pose prior weights does not match the' +
' number of face joint distance weights')
assert (len(face_joints_weights) ==
len(body_pose_prior_weights)), msg
if coll_loss_weights is None:
coll_loss_weights = [0.0] * len(body_pose_prior_weights)
msg = ('Number of Body pose prior weights does not match the' +
' number of collision loss weights')
assert (len(coll_loss_weights) ==
len(body_pose_prior_weights)), msg
use_vposer = kwargs.get('use_vposer', True)
vposer, pose_embedding = [None, ] * 2
if use_vposer:
pose_embedding = torch.zeros([batch_size, 32],
dtype=dtype, device=device,
requires_grad=True)
vposer_ckpt = osp.expandvars(vposer_ckpt)
vposer, _ = load_vposer(vposer_ckpt, vp_model='snapshot')
vposer = vposer.to(device=device)
vposer.eval()
if use_vposer:
body_mean_pose = torch.zeros([batch_size, vposer_latent_dim],
dtype=dtype)
else:
body_mean_pose = body_pose_prior.get_mean().detach().cpu()
keypoint_data = torch.tensor(keypoints, dtype=dtype)
gt_joints = keypoint_data[:, :, :2]
if use_joints_conf:
joints_conf = keypoint_data[:, :, 2].reshape(1, -1)
# Transfer the data to the correct device
gt_joints = gt_joints.to(device=device, dtype=dtype)
if use_joints_conf:
joints_conf = joints_conf.to(device=device, dtype=dtype)
# Create the search tree
search_tree = None
pen_distance = None
filter_faces = None
if interpenetration:
from mesh_intersection.bvh_search_tree import BVH
import mesh_intersection.loss as collisions_loss
from mesh_intersection.filter_faces import FilterFaces
assert use_cuda, 'Interpenetration term can only be used with CUDA'
assert torch.cuda.is_available(), \
'No CUDA Device! Interpenetration term can only be used' + \
' with CUDA'
search_tree = BVH(max_collisions=max_collisions)
pen_distance = \
collisions_loss.DistanceFieldPenetrationLoss(
sigma=df_cone_height, point2plane=point2plane,
vectorized=True, penalize_outside=penalize_outside)
if part_segm_fn:
# Read the part segmentation
part_segm_fn = os.path.expandvars(part_segm_fn)
with open(part_segm_fn, 'rb') as faces_parents_file:
face_segm_data = pickle.load(faces_parents_file,
encoding='latin1')
faces_segm = face_segm_data['segm']
faces_parents = face_segm_data['parents']
# Create the module used to filter invalid collision pairs
filter_faces = FilterFaces(
faces_segm=faces_segm, faces_parents=faces_parents,
ign_part_pairs=ign_part_pairs).to(device=device)
# Weights used for the pose prior and the shape prior
opt_weights_dict = {'data_weight': data_weights,
'body_pose_weight': body_pose_prior_weights,
'shape_weight': shape_weights}
if use_face:
opt_weights_dict['face_weight'] = face_joints_weights
opt_weights_dict['expr_prior_weight'] = expr_weights
opt_weights_dict['jaw_prior_weight'] = jaw_pose_prior_weights
if use_hands:
opt_weights_dict['hand_weight'] = hand_joints_weights
opt_weights_dict['hand_prior_weight'] = hand_pose_prior_weights
if interpenetration:
opt_weights_dict['coll_loss_weight'] = coll_loss_weights
keys = opt_weights_dict.keys()
opt_weights = [dict(zip(keys, vals)) for vals in
zip(*(opt_weights_dict[k] for k in keys
if opt_weights_dict[k] is not None))]
for weight_list in opt_weights:
for key in weight_list:
weight_list[key] = torch.tensor(weight_list[key],
device=device,
dtype=dtype)
# The indices of the joints used for the initialization of the camera
init_joints_idxs = torch.tensor(init_joints_idxs, device=device)
edge_indices = kwargs.get('body_tri_idxs')
init_t = fitting.guess_init(body_model, gt_joints, edge_indices,
use_vposer=use_vposer, vposer=vposer,
pose_embedding=pose_embedding,
model_type=kwargs.get('model_type', 'smpl'),
focal_length=focal_length, dtype=dtype)
camera_loss = fitting.create_loss('camera_init',
trans_estimation=init_t,
init_joints_idxs=init_joints_idxs,
depth_loss_weight=depth_loss_weight,
dtype=dtype).to(device=device)
camera_loss.trans_estimation[:] = init_t
loss = fitting.create_loss(loss_type=loss_type,
joint_weights=joint_weights,
rho=rho,
use_joints_conf=use_joints_conf,
use_face=use_face, use_hands=use_hands,
vposer=vposer,
pose_embedding=pose_embedding,
body_pose_prior=body_pose_prior,
shape_prior=shape_prior,
angle_prior=angle_prior,
expr_prior=expr_prior,
left_hand_prior=left_hand_prior,
right_hand_prior=right_hand_prior,
jaw_prior=jaw_prior,
interpenetration=interpenetration,
pen_distance=pen_distance,
search_tree=search_tree,
tri_filtering_module=filter_faces,
dtype=dtype,
**kwargs)
loss = loss.to(device=device)
print("tipe img")
print(type(img))
with fitting.FittingMonitor(
batch_size=batch_size, visualize=visualize, background_image = img, camera = camera, **kwargs) as monitor:
img = torch.tensor(img, dtype=dtype)
H, W, _ = img.shape
data_weight = 1000 / H
# The closure passed to the optimizer
camera_loss.reset_loss_weights({'data_weight': data_weight})
# Reset the parameters to estimate the initial translation of the
# body model
body_model.reset_params(body_pose=body_mean_pose)
# If the distance between the 2D shoulders is smaller than a
# predefined threshold then try 2 fits, the initial one and a 180
# degree rotation
shoulder_dist = torch.dist(gt_joints[:, left_shoulder_idx],
gt_joints[:, right_shoulder_idx])
try_both_orient = shoulder_dist.item() < side_view_thsh
# Update the value of the translation of the camera as well as
# the image center.
with torch.no_grad():
camera.translation[:] = init_t.view_as(camera.translation)
camera.center[:] = torch.tensor([W, H], dtype=dtype) * 0.5
# Re-enable gradient calculation for the camera translation
camera.translation.requires_grad = True
camera_opt_params = [camera.translation, body_model.global_orient]
camera_optimizer, camera_create_graph = optim_factory.create_optimizer(
camera_opt_params,
**kwargs)
# The closure passed to the optimizer
fit_camera = monitor.create_fitting_closure(
camera_optimizer, body_model, camera, gt_joints,
camera_loss, create_graph=camera_create_graph,
use_vposer=use_vposer, vposer=vposer,
pose_embedding=pose_embedding,
return_full_pose=False, return_verts=False)
# Step 1: Optimize over the torso joints the camera translation
# Initialize the computational graph by feeding the initial translation
# of the camera and the initial pose of the body model.
camera_init_start = time.time()
cam_init_loss_val = monitor.run_fitting(camera_optimizer,
fit_camera,
camera_opt_params, body_model,
use_vposer=use_vposer,
pose_embedding=pose_embedding,
vposer=vposer)
if interactive:
if use_cuda and torch.cuda.is_available():
torch.cuda.synchronize()
tqdm.write('Camera initialization done after {:.4f}'.format(
time.time() - camera_init_start))
tqdm.write('Camera initialization final loss {:.4f}'.format(
cam_init_loss_val))
# If the 2D detections/positions of the shoulder joints are too
# close the rotate the body by 180 degrees and also fit to that
# orientation
if try_both_orient:
body_orient = body_model.global_orient.detach().cpu().numpy()
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
flipped_orient = torch.tensor(flipped_orient,
dtype=dtype,
device=device).unsqueeze(dim=0)
orientations = [body_orient, flipped_orient]
else:
orientations = [body_model.global_orient.detach().cpu().numpy()]
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
results = []
# Step 2: Optimize the full model
final_loss_val = 0
for or_idx, orient in enumerate(tqdm(orientations, desc='Orientation')):
opt_start = time.time()
new_params = defaultdict(global_orient=orient,
body_pose=body_mean_pose)
body_model.reset_params(**new_params)
if use_vposer:
with torch.no_grad():
pose_embedding.fill_(0)
for opt_idx, curr_weights in enumerate(tqdm(opt_weights, desc='Stage')):
body_params = list(body_model.parameters())
final_params = list(
filter(lambda x: x.requires_grad, body_params))
if use_vposer:
final_params.append(pose_embedding)
body_optimizer, body_create_graph = optim_factory.create_optimizer(
final_params,
**kwargs)
body_optimizer.zero_grad()
curr_weights['data_weight'] = data_weight
curr_weights['bending_prior_weight'] = (
3.17 * curr_weights['body_pose_weight'])
if use_hands:
joint_weights[:, 25:76] = curr_weights['hand_weight']
if use_face:
joint_weights[:, 76:] = curr_weights['face_weight']
loss.reset_loss_weights(curr_weights)
closure = monitor.create_fitting_closure(
body_optimizer, body_model,
camera=camera, gt_joints=gt_joints,
joints_conf=joints_conf,
joint_weights=joint_weights,
loss=loss, create_graph=body_create_graph,
use_vposer=use_vposer, vposer=vposer,
pose_embedding=pose_embedding,
return_verts=True, return_full_pose=True)
if interactive:
if use_cuda and torch.cuda.is_available():
torch.cuda.synchronize()
stage_start = time.time()
final_loss_val = monitor.run_fitting(
body_optimizer,
closure, final_params,
body_model,
pose_embedding=pose_embedding, vposer=vposer,
use_vposer=use_vposer)
if interactive:
if use_cuda and torch.cuda.is_available():
torch.cuda.synchronize()
elapsed = time.time() - stage_start
if interactive:
tqdm.write('Stage {:03d} done after {:.4f} seconds'.format(
opt_idx, elapsed))
if interactive:
if use_cuda and torch.cuda.is_available():
torch.cuda.synchronize()
elapsed = time.time() - opt_start
tqdm.write(
'Body fitting Orientation {} done after {:.4f} seconds'.format(
or_idx, elapsed))
tqdm.write('Body final loss val = {:.5f}'.format(
final_loss_val))
# Get the result of the fitting process
# Store in it the errors list in order to compare multiple
# orientations, if they exist
result = {'camera_' + str(key): val.detach().cpu().numpy()
for key, val in camera.named_parameters()}
result.update({key: val.detach().cpu().numpy()
for key, val in body_model.named_parameters()})
if use_vposer:
result['body_pose'] = pose_embedding.detach().cpu().numpy()
results.append({'loss': final_loss_val,
'result': result})
with open(result_fn, 'wb') as result_file:
if len(results) > 1:
min_idx = (0 if results[0]['loss'] < results[1]['loss']
else 1)
else:
min_idx = 0
pickle.dump(results[min_idx]['result'], result_file, protocol=2)
if save_meshes or visualize:
body_pose = vposer.decode(
pose_embedding,
output_type='aa').view(1, -1) if use_vposer else None
model_type = kwargs.get('model_type', 'smpl')
append_wrists = model_type == 'smpl' and use_vposer
if append_wrists:
wrist_pose = torch.zeros([body_pose.shape[0], 6],
dtype=body_pose.dtype,
device=body_pose.device)
body_pose = torch.cat([body_pose, wrist_pose], dim=1)
model_output = body_model(return_verts=True, body_pose=body_pose)
vertices = model_output.vertices.detach().cpu().numpy().squeeze()
import trimesh
out_mesh = trimesh.Trimesh(vertices, body_model.faces)
rot = trimesh.transformations.rotation_matrix(
np.radians(180), [1, 0, 0])
out_mesh.apply_transform(rot)
out_mesh.export(mesh_fn)
visualize=True
if visualize:
import pyrender
material = pyrender.MetallicRoughnessMaterial(
metallicFactor=0.0,
alphaMode='OPAQUE',
baseColorFactor=(1.0, 1.0, 0.9, 1.0))
mesh = pyrender.Mesh.from_trimesh(
out_mesh,
material=material)
scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 0.0],
ambient_light=(0.3, 0.3, 0.3))
scene.add(mesh, 'mesh')
camera_center = camera.center.detach().cpu().numpy().squeeze()
camera_transl = camera.translation.detach().cpu().numpy().squeeze()
# Equivalent to 180 degrees around the y-axis. Transforms the fit to
# OpenGL compatible coordinate system.
camera_transl[0] *= -1.0
camera_pose = np.eye(4)
camera_pose[:3, 3] = camera_transl
camera = pyrender.camera.IntrinsicsCamera(
fx=focal_length, fy=focal_length,
cx=camera_center[0], cy=camera_center[1])
scene.add(camera, pose=camera_pose)
# # Get the lights from the viewer
light_nodes = create_raymond_lights()
for node in light_nodes:
scene.add_node(node)
r = pyrender.OffscreenRenderer(viewport_width=W,
viewport_height=H,
point_size=1.0)
color, _ = r.render(scene, flags=pyrender.RenderFlags.RGBA)
color = color.astype(np.float32) / 255.0
valid_mask = (color[:, :, -1] > 0)[:, :, np.newaxis]
input_img = img.detach().cpu().numpy()
output_img = (color[:, :, :-1] * valid_mask +
(1 - valid_mask) * input_img)
img = pil_img.fromarray((output_img * 255).astype(np.uint8))
print(out_img_fn)
print(img)
img.save(out_img_fn)
def test_simple_input_with_scalar_baseline_conductance(self) -> None:
net = BasicModel_MultiLayer()
inp = torch.tensor([[0.0, 100.0, 0.0]])
self._conductance_test_assert(
net, net.linear0, inp, [[0.0, 390.0, 0.0]], baselines=0.0
)
def create_dataset(args, processor, retrievers, relation_list, evaluate, input_dir):
if args.local_rank not in [-1, 0]:
# Make sure only the first process in distributed training process the dataset, and the others will use the cache
torch.distributed.barrier()
definition_info = DefinitionInfo()
tokenizer, _ = configure_tokenizer_model(args, logger, retrievers, is_preprocess=True)
logger.info("tokenizer: {}".format(tokenizer))
if args.test:
temp_mark = "test"
elif evaluate:
temp_mark = "dev"
else:
temp_mark = "train"
cached_features_file = os.path.join(
input_dir,
"cached_{}_{}_{}".format(
temp_mark,
args.model_type,
str(args.cache_file_suffix),
),
)
if os.path.exists(cached_features_file):
logger.warning("cache file exist and exit program")
exit()
logger.info("Creating features from dataset file at %s", input_dir)
if not os.path.exists(cached_features_file + "_example"):
if args.test:
examples = processor.get_test_examples(args.data_dir, filename=args.predict_file)
else:
examples = processor.get_dev_examples(args.data_dir, filename=args.predict_file)
torch.save(examples, cached_features_file + "_example")
else:
logger.info("Loading examples from cached files.")
examples = torch.load(cached_features_file + "_example")
examples_tokenized = processor.tokenization_on_examples(examples, tokenizer, is_testing=args.test)
features = processor.convert_examples_to_features(args, examples_tokenized, tokenizer, retrievers, not evaluate, debug=args.debug)
features, dataset, all_kgs_graphs = processor.pad_and_index_features_all(
features, retrievers, args, tokenizer, relation_list, encoder=None, definition_info=definition_info, is_training=not evaluate, debug=args.debug)
if args.local_rank in [-1, 0]:
for num in range(2):
all_kgs_graphs_label_dict = {"glabel": torch.tensor([i for i in range(len(all_kgs_graphs))])}
tmp_all_kgs_graphs = [gg[num] for gg in all_kgs_graphs]
save_graphs(cached_features_file + "_all_kgs_graphs_choice_{}.bin".format(num), tmp_all_kgs_graphs,
all_kgs_graphs_label_dict)
logger.info("complete data preprocessing")
logger.info("Saving features into cached file %s", cached_features_file)
for f in features:
del f.kgs_conceptids2synset_list
torch.save({"features": features, "dataset": dataset, "examples": examples_tokenized}, cached_features_file)
logger.info("Saving knowledge graph retrievers")
for kg, retriever in retrievers.items():
if not os.path.exists(os.path.join(input_dir, args.kg_paths[kg])):
os.mkdir(os.path.join(input_dir, args.kg_paths[kg]))
torch.save(retriever, os.path.join(input_dir, args.kg_paths[kg], kg + args.cache_file_suffix))
logger.info("data create is done")
if args.local_rank == 0:
# Make sure only the first process in distributed training process the dataset, and the others will use the cache
torch.distributed.barrier()
def __init__(
self,
n_input: int,
n_batch: int = 0,
n_labels: int = 0,
n_hidden: int = 128,
n_latent: int = 10,
n_layers: int = 1,
dropout_rate: float = 0.1,
dispersion: str = "gene",
log_variational: bool = True,
gene_likelihood: str = "zinb",
y_prior=None,
labels_groups: Sequence[int] = None,
use_labels_groups: bool = False,
classifier_parameters: dict = dict(),
):
super().__init__(
n_input,
n_hidden=n_hidden,
n_latent=n_latent,
n_layers=n_layers,
dropout_rate=dropout_rate,
n_batch=n_batch,
dispersion=dispersion,
log_variational=log_variational,
gene_likelihood=gene_likelihood,
)
self.n_labels = n_labels
# Classifier takes n_latent as input
cls_parameters = {
"n_layers": n_layers,
"n_hidden": n_hidden,
"dropout_rate": dropout_rate,
}
cls_parameters.update(classifier_parameters)
self.classifier = Classifier(n_latent,
n_labels=n_labels,
**cls_parameters)
self.encoder_z2_z1 = Encoder(
n_latent,
n_latent,
n_cat_list=[self.n_labels],
n_layers=n_layers,
n_hidden=n_hidden,
dropout_rate=dropout_rate,
)
self.decoder_z1_z2 = Decoder(
n_latent,
n_latent,
n_cat_list=[self.n_labels],
n_layers=n_layers,
n_hidden=n_hidden,
)
self.y_prior = torch.nn.Parameter(
y_prior if y_prior is not None else
(1 / n_labels) * torch.ones(1, n_labels),
requires_grad=False,
)
self.use_labels_groups = use_labels_groups
self.labels_groups = (np.array(labels_groups)
if labels_groups is not None else None)
if self.use_labels_groups:
assert labels_groups is not None, "Specify label groups"
unique_groups = np.unique(self.labels_groups)
self.n_groups = len(unique_groups)
assert (unique_groups == np.arange(self.n_groups)).all()
self.classifier_groups = Classifier(n_latent, n_hidden,
self.n_groups, n_layers,
dropout_rate)
self.groups_index = torch.nn.ParameterList([
torch.nn.Parameter(
torch.tensor(
(self.labels_groups == i).astype(np.uint8),
dtype=torch.uint8,
),
requires_grad=False,
) for i in range(self.n_groups)
])
def train_one_epoch(
cls,
model: torch.nn.Module,
iterator: Iterable[Tuple[List[str], Dict[str, torch.Tensor]]],
optimizers: Sequence[torch.optim.Optimizer],
schedulers: Sequence[Optional[AbsScheduler]],
reporter: SubReporter,
options: TrainerOptions,
) -> bool:
assert check_argument_types()
# Note(kamo): assumes one optimizer
assert cls.num_optimizers == 1, cls.num_optimizers
assert len(optimizers) == 1, len(optimizers)
optimizer = optimizers[0]
scheduler = schedulers[0]
grad_noise = options.grad_noise
accum_grad = options.accum_grad
grad_clip = options.grad_clip
log_interval = options.log_interval
no_forward_run = options.no_forward_run
ngpu = options.ngpu
distributed = isinstance(model,
torch.nn.parallel.DistributedDataParallel)
use_apex = options.train_dtype in ("O0", "O1", "O2", "O3")
if log_interval is None:
try:
log_interval = max(len(iterator) // 20, 10)
except TypeError:
log_interval = 100
model.train()
all_steps_are_invalid = True
# [For distributed] Because iteration counts are not always equals between
# processes, send stop-flag to the other processes if iterator is finished
iterator_stop = torch.tensor(0).to("cuda" if ngpu > 0 else "cpu")
start_time = time.perf_counter()
for iiter, (_, batch) in enumerate(
reporter.measure_iter_time(iterator, "iter_time"), 1):
assert isinstance(batch, dict), type(batch)
if distributed:
torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM)
if iterator_stop > 0:
break
batch = to_device(batch, "cuda" if ngpu > 0 else "cpu")
if no_forward_run:
all_steps_are_invalid = False
reporter.register({})
continue
with reporter.measure_time("forward_time"):
loss, stats, weight = model(**batch)
if ngpu > 1 or distributed:
# Apply weighted averaging for loss and stats
loss = (loss * weight.type(loss.dtype)).sum()
# if distributed, this method can also apply all_reduce()
stats, weight = recursive_average(stats, weight, distributed)
# Now weight is summation over all workers
loss /= weight
if distributed:
# NOTE(kamo): Multiply world_size because DistributedDataParallel
# automatically normalizes the gradient by world_size.
loss *= torch.distributed.get_world_size()
reporter.register(stats, weight)
loss /= accum_grad
with reporter.measure_time("backward_time"):
if use_apex:
try:
from apex import amp
except ImportError:
logging.error(
"You need to install apex. "
"See https://github.com/NVIDIA/apex#linux")
with amp.scale_loss(loss, optimizers) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if iiter % accum_grad == 0:
# gradient noise injection
if grad_noise:
add_gradient_noise(
model,
reporter.get_total_count(),
duration=100,
eta=1.0,
scale_factor=0.55,
)
# compute the gradient norm to check if it is normal or not
grad_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), grad_clip)
# PyTorch<=1.4, clip_grad_norm_ returns float value
if not isinstance(grad_norm, torch.Tensor):
grad_norm = torch.tensor(grad_norm)
if not torch.isfinite(grad_norm):
logging.warning(
f"The grad norm is {grad_norm}. Skipping updating the model."
)
else:
all_steps_are_invalid = False
with reporter.measure_time("optim_step_time"):
optimizer.step()
if isinstance(scheduler, AbsBatchStepScheduler):
scheduler.step()
optimizer.zero_grad()
# Register lr and train/load time[sec/step],
# where step refers to accum_grad * mini-batch
reporter.register(
dict(
{
f"lr_{i}": pg["lr"]
for i, pg in enumerate(optimizer.param_groups)
if "lr" in pg
},
train_time=time.perf_counter() - start_time,
),
# Suppress to increment the internal counter.
not_increment_count=True,
)
start_time = time.perf_counter()
if iiter % log_interval == 0:
logging.info(reporter.log_message())
else:
if distributed:
iterator_stop.fill_(1)
torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM)
return all_steps_are_invalid
def preprocess(self, state):
state = torch.tensor(np.expand_dims(state, 0)).to(self.device)
return state.float() / 256
print("done")
print(len(index2word))
print(len(word2index))
unkvec = word2vec["the"].copy()
padvec = word2vec["apple"].copy()
random.shuffle(unkvec)
random.shuffle(padvec)
avec = word2vec["a"]
at = torch.tensor([avec])
print(at.size())
l = []
for index in range(0, len(word2vec)):
if index % 100000 == 0:
print(index)
vector = word2vec[index2word[index]]
l.append(vector)
l.append(unkvec)
l.append(padvec)
embmatrix = torch.tensor(l)
def _images_to_observation(images, bit_depth): images = torch.tensor(cv2.resize(images, (64, 64), interpolation=cv2.INTER_LINEAR).transpose(2, 0, 1), dtype=torch.float32) # Resize and put channel first preprocess_observation_(images, bit_depth) # Quantise, centre and dequantise inplace return images.unsqueeze(dim=0) # Add batch dimension
out = z.mean()
print(z, out)
a = torch.randn(2, 2)
a = ((a * 3) / (a - 1))
print(a.requires_grad)
a.requires_grad_(True)
print(a.requires_grad)
b = (a * a).sum()
print(b.grad_fn)
print(x.grad)
out.backward()
print(x.grad)
x = torch.randn(3, requires_grad=True)
y = x * 2
while y.data.norm() < 1000:
y = y * 2
print(y)
gradients = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)
y.backward(gradients)
print(x.grad)
print(x.requires_grad)
print((x ** 2).requires_grad)
with torch.no_grad():
print((x ** 2).requires_grad)
def __init__(self, loss, weight=1.0):
super().__init__()
self.loss = loss
self.register_buffer("weight", torch.tensor([weight]))
def train_student(
self,
epochs=10,
plot_losses=True,
save_model=True,
save_model_pth="./models/student.pth",
):
"""
Function that will be training the student
:param epochs (int): Number of epochs you want to train the teacher
:param plot_losses (bool): True if you want to plot the losses
:param save_model (bool): True if you want to save the student model
:param save_model_pth (str): Path where you want to save the student model
"""
self.teacher_distill_loader = self._get_teacher_dataloaders(
batch_size=self.train_loader.batch_size, mode="distill")
y_pred_teacher = []
print("Obtaining teacher predictions...")
self.teacher_model.eval()
self.teacher_model.to(self.device)
for batch in self.teacher_distill_loader:
b_input_ids = batch[0].to(self.device)
b_input_mask = batch[1].to(self.device)
b_labels = batch[2].to(self.device)
with torch.no_grad():
(loss, logits) = self.teacher_model(
b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels,
)
logits = logits.detach().cpu().numpy()
y_pred_teacher.append(logits)
self.student_model.train()
loss_arr = []
length_of_dataset = len(self.train_loader.dataset)
best_acc = 0.0
self.best_student_model_weights = deepcopy(
self.student_model.state_dict())
self.student_model.to(self.device)
print("\nTraining student...")
for ep in range(epochs):
print("")
print("======== Epoch {:} / {:} ========".format(ep + 1, epochs))
epoch_loss = 0.0
correct = 0
for (data, data_len,
label), bert_prob in zip(self.train_loader, y_pred_teacher):
data = data.to(self.device)
data_len = data_len.to(self.device)
label = label.to(self.device)
bert_prob = torch.tensor(bert_prob, dtype=torch.float)
teacher_out = bert_prob.to(self.device)
self.optimizer_student.zero_grad()
student_out = self.student_model(data, data_len).squeeze(1)
loss = self.calculate_kd_loss(student_out, teacher_out, label)
pred = student_out.argmax(dim=1, keepdim=True)
correct += pred.eq(label.view_as(pred)).sum().item()
loss.backward()
# ##For preventing exploding gradients
# torch.nn.utils.clip_grad_norm_(self.student_model.parameters(), 1.0)
self.optimizer_student.step()
epoch_loss += loss
epoch_acc = correct / length_of_dataset
print(f"Loss: {epoch_loss} | Accuracy: {epoch_acc}")
epoch_acc = correct / length_of_dataset
if epoch_acc > best_acc:
best_acc = epoch_acc
self.best_student_model_weights = deepcopy(
self.student_model.state_dict())
if self.log:
self.writer.add_scalar("Training loss/Student", epoch_loss,
epochs)
self.writer.add_scalar("Training accuracy/Student", epoch_acc,
epochs)
loss_arr.append(epoch_loss)
print(f"Epoch: {ep+1}, Loss: {epoch_loss}, Accuracy: {epoch_acc}")
self.student_model.load_state_dict(self.best_student_model_weights)
if save_model:
torch.save(self.student_model.state_dict(), save_model_pth)
if plot_losses:
plt.plot(loss_arr)
def test_simple_input_conductance(self):
net = BasicModel_MultiLayer()
inp = torch.tensor([[0.0, 100.0, 0.0]])
self._conductance_test_assert(net, net.linear0, inp,
[[0.0, 390.0, 0.0]])
def step(self, actions): observations, rewards, dones = zip(*[env.step(action) for env, action in zip(self.envs, actions)]) self.dones = dones return torch.cat(observations), torch.tensor(rewards, dtype=torch.float32), torch.tensor(dones, dtype=torch.uint8)
