def train_batch(param): if len(memory) < param['batch_size']: return 0 batch = memory.sample(param['batch_size']) batch_states = default_states_preprocessor([m.state for m in batch]) batch_next_states = default_states_preprocessor([m.next_state for m in batch]) batch_ended = torch.tensor([m.ended for m in batch]) batch_rewards = torch.tensor([m.reward for m in batch]).to(device) batch_actions = torch.tensor([m.action for m in batch]).to(device) ## Calculate expected reward: with torch.set_grad_enabled(False): not_ended_batch = 1 -torch.ByteTensor(batch_ended).to(device) next_states_non_final = batch_next_states[not_ended_batch] next_state_values = torch.zeros(param['batch_size']).to(device) reward_hat = target_dqn(next_states_non_final) next_state_values[not_ended_batch] = reward_hat.max(1)[0] expected_state_action_values = next_state_values*param['GAMMA'] + batch_rewards # Predict value function: yhat = dqn(batch_states) state_action_values = yhat.gather(1, batch_actions.unsqueeze(1)).squeeze() loss = F.smooth_l1_loss(state_action_values, expected_state_action_values) optimizer.zero_grad() loss.backward() for param in dqn.parameters(): param.data.clamp_(-1, 1) optimizer.step() return float(loss.data.cpu().numpy())
def test_manual_bounds(self, cuda=False): device = torch.device("cuda") if cuda else torch.device("cpu") for dtype in (torch.float, torch.double): # get a test module train_x = torch.tensor([[1.0, 2.0, 3.0]], device=device, dtype=dtype) train_y = torch.tensor([4.0], device=device, dtype=dtype) likelihood = GaussianLikelihood() model = ExactGP(train_x, train_y, likelihood) model.covar_module = RBFKernel(ard_num_dims=3) model.mean_module = ConstantMean() model.to(device=device, dtype=dtype) mll = ExactMarginalLogLikelihood(likelihood, model) # test the basic case x, pdict, bounds = module_to_array( module=mll, bounds={"model.covar_module.raw_lengthscale": (0.1, None)} ) self.assertTrue(np.array_equal(x, np.zeros(5))) expected_sizes = { "likelihood.noise_covar.raw_noise": torch.Size([1]), "model.covar_module.raw_lengthscale": torch.Size([1, 3]), "model.mean_module.constant": torch.Size([1]), } self.assertEqual(set(pdict.keys()), set(expected_sizes.keys())) for pname, val in pdict.items(): self.assertEqual(val.dtype, dtype) self.assertEqual(val.shape, expected_sizes[pname]) self.assertEqual(val.device.type, device.type) lower_exp = np.full_like(x, 0.1) for p in ("likelihood.noise_covar.raw_noise", "model.mean_module.constant"): lower_exp[_get_index(pdict, p)] = -np.inf self.assertTrue(np.equal(bounds[0], lower_exp).all()) self.assertTrue(np.equal(bounds[1], np.full_like(x, np.inf)).all())
def test_factory(self): default_size = torch.Size([1, 3]) size = torch.Size([3, 3]) for include_size in [True, False]: for use_tensor_idx in [True, False]: for use_tensor_val in [True, False]: for use_cuda in ([False] if not torch.cuda.is_available() else [True, False]): # have to include size with cuda sparse tensors include_size = include_size or use_cuda dtype = torch.float64 long_dtype = torch.int64 device = torch.device('cpu') if not use_cuda else torch.device(torch.cuda.device_count() - 1) indices = torch.tensor(([0], [2]), dtype=long_dtype) if use_tensor_idx else ([0], [2]) values = torch.tensor([1.], dtype=dtype) if use_tensor_val else 1. if include_size: sparse_tensor = torch.sparse_coo_tensor(indices, values, size, dtype=dtype, device=device, requires_grad=True) else: sparse_tensor = torch.sparse_coo_tensor(indices, values, dtype=dtype, device=device, requires_grad=True) self.assertEqual(indices, sparse_tensor._indices()) self.assertEqual(values, sparse_tensor._values()) self.assertEqual(size if include_size else default_size, sparse_tensor.size()) self.assertEqual(dtype, sparse_tensor.dtype) if use_cuda: self.assertEqual(device, sparse_tensor._values().device) self.assertEqual(True, sparse_tensor.requires_grad)
def test_cases_cos(): a=torch.tensor([[1,2,3.0],[0,0,0.0]]) b=torch.tensor([[1,2,3.1],[-1,-2,-3.0]]) a2=torch.tensor([[1,2,3.0],[0,0,0]]) b2=torch.tensor([[1,2,3.1],[-1,-2,-3.0],[5,5,5.0],[6,6,6.0]]) a3=torch.tensor([1,2,3.0]) b3=torch.tensor([1,2,3.1]) a31=torch.tensor([[1,2,3.0]]) b31=torch.tensor([[1,2,3.1]]) ar=np.random.rand(5,10) br=np.random.rand(15,10) art=torch.tensor(ar) brt=torch.tensor(br) cos(a,b) cos(a2,b2) abrt = cos(art,brt) print("sklearn cos:", sklearn.metrics.pairwise.cosine_similarity(ar,br)) cos(a3,b3) try: cos(a3,b3) except: print("cos(a3,b3) failed") try: cos([a3],[b3]) except: print("cos(a3,b3) failed") cos(a31,b31)
def test_index_setitem_bools_slices(self): true = torch.tensor(1, dtype=torch.uint8) false = torch.tensor(0, dtype=torch.uint8) tensors = [Variable(torch.randn(2, 3)), torch.tensor(3)] for a in tensors: # prefix with a 1,1, to ensure we are compatible with numpy which cuts off prefix 1s # (some of these ops already prefix a 1 to the size) neg_ones = torch.ones_like(a) * -1 neg_ones_expanded = neg_ones.unsqueeze(0).unsqueeze(0) a[True] = neg_ones_expanded self.assertEqual(a, neg_ones) a[False] = 5 self.assertEqual(a, neg_ones) a[true] = neg_ones_expanded * 2 self.assertEqual(a, neg_ones * 2) a[false] = 5 self.assertEqual(a, neg_ones * 2) a[None] = neg_ones_expanded * 3 self.assertEqual(a, neg_ones * 3) a[...] = neg_ones_expanded * 4 self.assertEqual(a, neg_ones * 4) if a.dim() == 0: with self.assertRaises(RuntimeError): a[:] = neg_ones_expanded * 5
def forward(self, sents, sent_lengths): ''' sents is (batch_size by padded_length) when we evaluate sentence by sentence, you evaluate it with batch_size = 1, padded_length. [[1, 2, 3, 4]] etc. ''' batch_size = sents.size()[0] sent_lengths = list(sent_lengths) # We sort and then do pad packed sequence here. descending_lengths = [x for x, _ in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)] descending_indices = [x for _, x in sorted(zip(sent_lengths, range(len(sent_lengths))), reverse=True)] descending_lengths = torch.tensor(descending_lengths) descending_indices = torch.tensor(descending_indices).to(device) descending_sents = torch.index_select(sents, torch.tensor(0), descending_indices) # get embedding embed = self.embedding(descending_sents) # pack padded sequence embed = torch.nn.utils.rnn.pack_padded_sequence(embed, descending_lengths, batch_first=True) # fprop though RNN self.hidden = self.init_hidden(batch_size) rnn_out, self.hidden = self.gru(embed, self.hidden) pdb.set_trace() rnn_out, _ = torch.nn.utils.rnn.pad_packed_sequence(rnn_out, batch_first=True) # rnn_out is 32 by 72 by 256 # change the order back change_it_back = [x for _, x in sorted(zip(descending_indices, range(len(descending_indices))))] self.hidden = torch.index_select(self.hidden, 1, torch.LongTensor(change_it_back).to(device)) rnn_out = torch.index_select(rnn_out, 0, torch.LongTensor(change_it_back).to(device)) return rnn_out, self.hidden
def generate_translation(encoder, decoder, sentence, max_length, target_lang, search="greedy", k = None): """ @param max_length: the max # of words that the decoder can return @returns decoded_words: a list of words in target language """ with torch.no_grad(): input_tensor = sentence input_length = sentence.size()[1] # encode the source sentence encoder_hidden = encoder.init_hidden(1) # input_tensor 1 by 12 # encoder_outputs, encoder_hidden = encoder(input_tensor.view(1, -1),torch.tensor([input_length])) # start decoding decoder_input = torch.tensor([[SOS_token]], device=device) # SOS decoder_hidden = encoder_hidden decoded_words = [] if search == 'greedy': decoded_words = greedy_search_batch(decoder, decoder_input, encoder_outputs, decoder_hidden, max_length) elif search == 'beam': if k == None: k = 2 # since k = 2 preforms badly decoded_words = beam_search(decoder, decoder_input, encoder_outputs, decoder_hidden, max_length, k, target_lang) return decoded_words
def __init__(self, mean, std): super(Normalization, self).__init__() # .view the mean and std to make them [C x 1 x 1] so that they can # directly work with image Tensor of shape [B x C x H x W]. # B is batch size. C is number of channels. H is height and W is width. self.mean = torch.tensor(mean).view(-1, 1, 1) self.std = torch.tensor(std).view(-1, 1, 1)
def perform_val(multi_gpu, device, embedding_size, batch_size, backbone, carray, issame, nrof_folds = 10, tta = True): if multi_gpu: backbone = backbone.module # unpackage model from DataParallel backbone = backbone.to(device) else: backbone = backbone.to(device) backbone.eval() # switch to evaluation mode idx = 0 embeddings = np.zeros([len(carray), embedding_size]) with torch.no_grad(): while idx + batch_size <= len(carray): batch = torch.tensor(carray[idx:idx + batch_size][:, [2, 1, 0], :, :]) if tta: fliped = hflip_batch(batch) emb_batch = backbone(batch.to(device)).cpu() + backbone(fliped.to(device)).cpu() embeddings[idx:idx + batch_size] = l2_norm(emb_batch) else: embeddings[idx:idx + batch_size] = backbone(batch.to(device)).cpu() idx += batch_size if idx < len(carray): batch = torch.tensor(carray[idx:]) if tta: fliped = hflip_batch(batch) emb_batch = backbone(batch.to(device)).cpu() + backbone(fliped.to(device)).cpu() embeddings[idx:] = l2_norm(emb_batch) else: embeddings[idx:] = backbone(batch.to(device)).cpu() tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds) buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) roc_curve_tensor = transforms.ToTensor()(roc_curve) return accuracy.mean(), best_thresholds.mean(), roc_curve_tensor
def calc_loss(batch, net, tgt_net, gamma, device="cpu", save_prefix=None): states, actions, rewards, dones, next_states = common.unpack_batch(batch) batch_size = len(batch) states_v = torch.tensor(states).to(device) actions_v = torch.tensor(actions).to(device) next_states_v = torch.tensor(next_states).to(device) # next state distribution next_distr_v, next_qvals_v = tgt_net.both(next_states_v) next_actions = next_qvals_v.max(1)[1].data.cpu().numpy() next_distr = tgt_net.apply_softmax(next_distr_v).data.cpu().numpy() next_best_distr = next_distr[range(batch_size), next_actions] dones = dones.astype(np.bool) # project our distribution using Bellman update proj_distr = common.distr_projection(next_best_distr, rewards, dones, Vmin, Vmax, N_ATOMS, gamma) # calculate net output distr_v = net(states_v) state_action_values = distr_v[range(batch_size), actions_v.data] state_log_sm_v = F.log_softmax(state_action_values, dim=1) proj_distr_v = torch.tensor(proj_distr).to(device) if save_prefix is not None: pred = F.softmax(state_action_values, dim=1).data.cpu().numpy() save_transition_images(batch_size, pred, proj_distr, next_best_distr, dones, rewards, save_prefix) loss_v = -state_log_sm_v * proj_distr_v return loss_v.sum(dim=1).mean()
def load(self, fdata, use_char=False, n_context=1, max_len=10): sentences = self.preprocess(fdata) x, y, char_x, lens = [], [], [], [] for wordseq, tagseq in sentences: wiseq = [self.wdict.get(w, self.unk_wi) for w in wordseq] tiseq = [self.tdict[t] for t in tagseq] # 获取每个词汇的上下文 if n_context > 1: x.append(self.get_context(wiseq, n_context)) else: x.append(torch.tensor(wiseq, dtype=torch.long)) y.append(torch.tensor(tiseq, dtype=torch.long)) # 不足最大长度的部分用0填充 char_x.append(torch.tensor([ [self.cdict.get(c, self.unk_ci) for c in w[:max_len]] + [0] * (max_len - len(w)) for w in wordseq ])) lens.append(len(tiseq)) x = pad_sequence(x, True) y = pad_sequence(y, True) char_x = pad_sequence(char_x, True) lens = torch.tensor(lens) if use_char: dataset = TensorDataset(x, y, char_x, lens) else: dataset = TensorDataset(x, y, lens) return dataset
def test_advance_with_all_repeats_gets_blocked(self): # all beams repeat (beam >= 1 repeat dummy scores) beam_sz = 5 n_words = 100 repeat_idx = 47 ngram_repeat = 3 beam = Beam(beam_sz, 0, 1, 2, n_best=2, exclusion_tokens=set(), global_scorer=GlobalScorerStub(), block_ngram_repeat=ngram_repeat) for i in range(ngram_repeat + 4): # predict repeat_idx over and over again word_probs = torch.full((beam_sz, n_words), -float('inf')) word_probs[0, repeat_idx] = 0 attns = torch.randn(beam_sz) beam.advance(word_probs, attns) if i <= ngram_repeat: self.assertTrue( beam.scores.equal( torch.tensor( [0] + [-float('inf')] * (beam_sz - 1)))) else: self.assertTrue( beam.scores.equal(torch.tensor( [self.BLOCKED_SCORE] * beam_sz)))
def test_gather_extended_gold_tokens(self): vocab_size = self.model._target_vocab_size end_index = self.model._end_index pad_index = self.model._pad_index oov_index = self.model._oov_index tok_index = 6 # some other arbitrary token assert tok_index not in [end_index, pad_index, oov_index] # first sentence tokens: # 1: oov but not copied # 2: not oov and not copied # 3: not copied # 4: not copied # second sentence tokens: # 1: not oov and copied # 2: oov and copied # 3: not copied # 4: not copied # shape: (batch_size, target_sequence_length) target_tokens = torch.tensor([[oov_index, tok_index, end_index, pad_index], [tok_index, oov_index, tok_index, end_index]]) # shape: (batch_size, trimmed_source_length) source_token_ids = torch.tensor([[0, 1, 2, 3], [0, 1, 0, 2]]) # shape: (batch_size, target_sequence_length) target_token_ids = torch.tensor([[4, 5, 6, 7], [1, 0, 3, 4]]) # shape: (batch_size, target_sequence_length) result = self.model._gather_extended_gold_tokens(target_tokens, source_token_ids, target_token_ids) # shape: (batch_size, target_sequence_length) check = np.array([[oov_index, tok_index, end_index, pad_index], [tok_index, vocab_size, tok_index, end_index]]) np.testing.assert_array_equal(result.numpy(), check)
def guide(num_particles): q1 = pyro.param("q1", torch.tensor(pi1, requires_grad=True)) q2 = pyro.param("q2", torch.tensor(pi2, requires_grad=True)) with pyro.iarange("particles", num_particles): z = pyro.sample("z", dist.Normal(q2, 1.0).expand_by([num_particles])) zz = torch.exp(z) / (1.0 + torch.exp(z)) pyro.sample("y", dist.Bernoulli(q1 * zz))
def testBinaryEntropy(self): p = torch.tensor([0.1, 0.02, 0.99, 0.5, 0.75, 0.8, 0.3, 0.4, 0.0, 1.0]) entropy, entropySum = binaryEntropy(p) self.assertAlmostEqual(entropySum, 5.076676985, places=4) self.assertAlmostEqual(entropySum, entropy.sum(), places=4) self.assertAlmostEqual(entropy[0], 0.468995594, places=4) self.assertAlmostEqual(entropy[1], 0.141440543, places=4) self.assertAlmostEqual(entropy[2], 0.080793136, places=4) self.assertEqual(entropy[8], 0.0) self.assertEqual(entropy[9], 0.0) p = torch.tensor([0.25, 0.25, 0.25, 0.25]) entropy, entropySum = binaryEntropy(p) self.assertAlmostEqual(entropySum, 3.245112498, places=4) self.assertAlmostEqual(entropySum, entropy.sum(), places=4) p = torch.tensor([0.5, 0.5, 0.5, 0.5]) entropy, entropySum = binaryEntropy(p) self.assertAlmostEqual(entropySum, 4.0, places=4) self.assertAlmostEqual(entropySum, entropy.sum(), places=4) self.assertAlmostEqual(entropy[0], 1.0, places=4) self.assertAlmostEqual(entropy[1], 1.0, places=4) self.assertAlmostEqual(entropy[2], 1.0, places=4) self.assertAlmostEqual(entropy[3], 1.0, places=4)
def guide(): q1 = pyro.param("q1", torch.tensor(pi1, requires_grad=True)) q2 = pyro.param("q2", torch.tensor(pi2, requires_grad=True)) with pyro.iarange("particles", num_particles): y = pyro.sample("y", dist.Bernoulli(q1).expand_by([num_particles]), infer={"enumerate": enumerate1}) if include_z: pyro.sample("z", dist.Normal(q2 * y + 0.10, 1.0))
def model(num_particles): with pyro.iarange("particles", num_particles): q3 = pyro.param("q3", torch.tensor(pi3, requires_grad=True)) q4 = pyro.param("q4", torch.tensor(0.5 * (pi1 + pi2), requires_grad=True)) z = pyro.sample("z", dist.Normal(q3, 1.0).expand_by([num_particles])) zz = torch.exp(z) / (1.0 + torch.exp(z)) pyro.sample("y", dist.Bernoulli(q4 * zz))
def test_advance_with_all_repeats_gets_blocked(self): # all beams repeat (beam >= 1 repeat dummy scores) beam_sz = 5 n_words = 100 repeat_idx = 47 ngram_repeat = 3 for batch_sz in [1, 3]: beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), 0, 30, False, ngram_repeat, set(), torch.randint(0, 30, (batch_sz,)), False, 0.) for i in range(ngram_repeat + 4): # predict repeat_idx over and over again word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) word_probs[0::beam_sz, repeat_idx] = 0 attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i <= ngram_repeat: expected_scores = torch.tensor( [0] + [-float('inf')] * (beam_sz - 1))\ .repeat(batch_sz, 1) self.assertTrue(beam.topk_log_probs.equal(expected_scores)) else: self.assertTrue( beam.topk_log_probs.equal( torch.tensor(self.BLOCKED_SCORE) .repeat(batch_sz, beam_sz)))
def diamond_guide(dim): p0 = torch.tensor(math.exp(-0.70), requires_grad=True) p1 = torch.tensor(math.exp(-0.43), requires_grad=True) pyro.sample("a1", dist.Bernoulli(p0)) for i in pyro.irange("irange", dim): pyro.sample("b{}".format(i), dist.Bernoulli(p1)) pyro.sample("c1", dist.Bernoulli(p0))
def test_optimizers(factory): optim = factory() def model(loc, cov): x = pyro.param("x", torch.randn(2)) y = pyro.param("y", torch.randn(3, 2)) z = pyro.param("z", torch.randn(4, 2).abs(), constraint=constraints.greater_than(-1)) pyro.sample("obs_x", dist.MultivariateNormal(loc, cov), obs=x) with pyro.iarange("y_iarange", 3): pyro.sample("obs_y", dist.MultivariateNormal(loc, cov), obs=y) with pyro.iarange("z_iarange", 4): pyro.sample("obs_z", dist.MultivariateNormal(loc, cov), obs=z) loc = torch.tensor([-0.5, 0.5]) cov = torch.tensor([[1.0, 0.09], [0.09, 0.1]]) for step in range(100): tr = poutine.trace(model).get_trace(loc, cov) loss = -tr.log_prob_sum() params = {name: pyro.param(name).unconstrained() for name in ["x", "y", "z"]} optim.step(loss, params) for name in ["x", "y", "z"]: actual = pyro.param(name) expected = loc.expand(actual.shape) assert_equal(actual, expected, prec=1e-2, msg='{} in correct: {} vs {}'.format(name, actual, expected))
def run_episode(self, episode, steps_accumulated=0): start_time = time.time() observation = self.env.reset() state = torch.from_numpy(observation).to(self.config.device, dtype=torch.float32).unsqueeze(0) for step in range(MAX_STEPS): action = self.agent.get_action(state, step + steps_accumulated) observation_next, _, done, _ = self.env.step(action.item()) if done: state_next = None self.total_step = np.hstack((self.total_step[1:], step + 1)) if self.is_success_episode(step): reward = torch.tensor([1.0], dtype=torch.float32, device=self.config.device) else: reward = torch.tensor([-1.0], dtype=torch.float32, device=self.config.device) else: reward = torch.tensor([0.0], dtype=torch.float32, device=self.config.device) state_next = torch.from_numpy(observation_next).to(self.config.device, dtype=torch.float32).unsqueeze(0) self.agent.observe(state, action, state_next, reward) if step % self.config.replay_interval == 0: self.agent.learn(episode) state = state_next if done: elapsed_time = round(time.time() - start_time, 3) print('episode: {0}, steps: {1}, mean steps {2}, time: {3}'.format(episode, step, self.total_step.mean(), elapsed_time)) return step + 1 return MAX_STEPS
def model(): p2 = torch.tensor(torch.ones(2) / 2) p3 = torch.tensor(torch.ones(3) / 3) x2 = pyro.sample("x2", dist.OneHotCategorical(p2)) x3 = pyro.sample("x3", dist.OneHotCategorical(p3)) assert x2.shape == torch.Size([2]) + iarange_shape + p2.shape assert x3.shape == torch.Size([3, 1]) + iarange_shape + p3.shape
def model(): with pyro.iarange("num_particles", 10, dim=-3): with pyro.iarange("components", 2, dim=-1): p = pyro.sample("p", dist.Beta(torch.tensor(1.1), torch.tensor(1.1))) assert p.shape == torch.Size((10, 1, 2)) with pyro.iarange("data", data.shape[0], dim=-2): pyro.sample("obs", dist.Bernoulli(p), obs=data)
def testDutyCycleUpdate(self): """ Start with equal duty cycle, boost factor=0, k=4, batch size=2 """ x = self.x2 expected = torch.zeros_like(x) expected[0, 0, 1, 0] = 1.1 expected[0, 0, 1, 1] = 1.2 expected[0, 1, 0, 1] = 1.2 expected[0, 2, 1, 0] = 1.3 expected[1, 0, 0, 0] = 1.4 expected[1, 1, 0, 0] = 1.5 expected[1, 1, 0, 1] = 1.6 expected[1, 2, 1, 1] = 1.7 dutyCycle = torch.zeros((1, 3, 1, 1)) dutyCycle[:] = 1.0 / 3.0 updateDutyCycleCNN(expected, dutyCycle, 2, 2) newDuty = torch.tensor([1.5000, 1.5000, 1.0000]) / 4.0 diff = (dutyCycle.reshape(-1) - newDuty).abs().sum() self.assertLessEqual(diff, 0.001) dutyCycle[:] = 1.0 / 3.0 updateDutyCycleCNN(expected, dutyCycle, 4, 4) newDuty = torch.tensor([0.3541667, 0.3541667, 0.2916667]) diff = (dutyCycle.reshape(-1) - newDuty).abs().sum() self.assertLessEqual(diff, 0.001)
def postprocess_sequence(self, X): """Embed (variable-length) sequences Parameters ---------- X : list List of input sequences Returns ------- fX : numpy array Batch of sequence embeddings. """ lengths = torch.tensor([len(x) for x in X]) sorted_lengths, sort = torch.sort(lengths, descending=True) _, unsort = torch.sort(sort) sequences = [torch.tensor(X[i], dtype=torch.float32, device=self.device) for i in sort] padded = pad_sequence(sequences, batch_first=True, padding_value=0) packed = pack_padded_sequence(padded, sorted_lengths, batch_first=True) cpu = torch.device('cpu') fX = self.model(packed).detach().to(cpu).numpy() return fX[unsort]
def predict_spec_sliding_window(spectrogram, model, chunk_size=256, jump=128, hierarchical_model=None, hierarchy_threshold=15): """ Generate the prediction sequence for a full audio sequence using a sliding window. Slide the window by one spectrogram frame and pass each window through the given model. Compute the average over overlapping window predictions to get the final prediction. Allow for having a hierarchical model! If using a hierarchical model also save the model_0 predictions! Return: With Hierarchical Model - hierarchical predictions, model_0 predictions Solo Model - predictions """ # Get the number of frames in the full audio clip predictions = np.zeros(spectrogram.shape[0]) if hierarchical_model is not None: hierarchical_predictions = np.zeros(spectrogram.shape[0]) # Keeps track of the number of predictions made for a given # slice for final averaging! overlap_counts = np.zeros(spectrogram.shape[0]) # This is a bit janky but we will manually transform # each spectrogram chunk #spectrogram = torch.from_numpy(spectrogram).float() # Add a batch dim for the model! #spectrogram = torch.unsqueeze(spectrogram, 0) # Shape - (1, time, freq) # Added! spectrogram = np.expand_dims(spectrogram,axis=0) # For the sliding window we slide the window by one spectrogram # frame, determined by the hop size. spect_idx = 0 # The frame idx of the beginning of the current window i = 0 # How can I parralelize this shit?????? while spect_idx + chunk_size <= spectrogram.shape[1]: spect_slice = spectrogram[:, spect_idx: spect_idx + chunk_size, :] # Transform the slice - this is definitely sketchy!!!! spect_slice = (spect_slice - np.mean(spect_slice)) / np.std(spect_slice) spect_slice = torch.from_numpy(spect_slice).float() spect_slice = spect_slice.to(parameters.device) outputs = model(spect_slice) # Shape - (1, chunk_size, 1) compressed_out = outputs.view(-1, 1).squeeze() # Now check if we are running the hierarchical model if hierarchical_model is not None: chunk_preds = torch.sigmoid(compressed_out) binary_preds = torch.where(chunk_preds > parameters.THRESHOLD, torch.tensor(1.0).to(parameters.device), torch.tensor(0.0).to(parameters.device)) pred_counts = torch.sum(binary_preds) # Check if we need to run the second model hierarchical_compressed_out = compressed_out if pred_counts.item() >= hierarchy_threshold: hierarchical_outputs = hierarchical_model(spect_slice) hierarchical_compressed_out = hierarchical_outputs.view(-1, 1).squeeze() # Save the hierarchical model's output hierarchical_predictions[spect_idx: spect_idx + chunk_size] += hierarchical_compressed_out.cpu().detach().numpy() overlap_counts[spect_idx: spect_idx + chunk_size] += 1 # Save the model_0's output! predictions[spect_idx: spect_idx + chunk_size] += compressed_out.cpu().detach().numpy() spect_idx += jump i += 1 # Do the last one if it was not covered if (spect_idx - jump + chunk_size != spectrogram.shape[1]): #print ('One final chunk!') spect_slice = spectrogram[:, spect_idx: , :] # Transform the slice # Should use the function from the dataset!! spect_slice = (spect_slice - np.mean(spect_slice)) / np.std(spect_slice) spect_slice = torch.from_numpy(spect_slice).float() spect_slice = spect_slice.to(parameters.device) outputs = model(spect_slice) # Shape - (1, chunk_size, 1) # In the case of ResNet the output is forced to the chunk size compressed_out = outputs.view(-1, 1).squeeze()[:predictions[spect_idx: ].shape[0]] # Now check if we are running the hierarchical model if hierarchical_model is not None: chunk_preds = torch.sigmoid(compressed_out) binary_preds = torch.where(chunk_preds > parameters.THRESHOLD, torch.tensor(1.0).to(parameters.device), torch.tensor(0.0).to(parameters.device)) pred_counts = torch.sum(binary_preds) # Check if we need to run the second model hierarchical_compressed_out = compressed_out if pred_counts.item() >= hierarchy_threshold: hierarchical_outputs = hierarchical_model(spect_slice) hierarchical_compressed_out = hierarchical_outputs.view(-1, 1).squeeze()[:predictions[spect_idx: ].shape[0]] # Save the hierarchical model's output hierarchical_predictions[spect_idx: ] += hierarchical_compressed_out.cpu().detach().numpy() overlap_counts[spect_idx: ] += 1 # Save the model_0's output! predictions[spect_idx: ] += compressed_out.cpu().detach().numpy() # Average the predictions on overlapping frames predictions = predictions / overlap_counts if hierarchical_model is not None: hierarchical_predictions = hierarchical_predictions / overlap_counts # Get squashed [0, 1] predictions predictions = sigmoid(predictions) if hierarchical_model is not None: hierarchical_predictions = sigmoid(hierarchical_predictions) return hierarchical_predictions, predictions return predictions
def test_simple_relu_conductance(self): net = BasicModel_MultiLayer() inp = torch.tensor([[0.0, 100.0, 0.0]]) self._conductance_test_assert(net, net.relu, inp, [[90.0, 100.0, 100.0, 100.0]])
def test_simple_linear_conductance(self): net = BasicModel_MultiLayer() inp = torch.tensor([[0.0, 100.0, 0.0]], requires_grad=True) self._conductance_test_assert(net, net.linear1, inp, [[90.0, 100.0, 100.0, 100.0]])
def __getitem__(self, i): return torch.tensor(self.examples[i], dtype=torch.long)
def avg(self): d = torch.tensor(list(self.deque), dtype=torch.float32) return d.mean().item()
def median(self): # @property 是装饰器,这里可简单理解为增加median属性(只读) d = torch.tensor(list(self.deque)) return d.median().item()
def get_sample(self, idx): assert type(idx) is int data = self.__getitem__(idx) return [torch.tensor(_dt.reshape(1, -1)) for _dt in data]
def generateTestCase (img, targetedNeuronIndex, desiredNAP, model): """ Generate a test case to satisfy the specified neuronActivationPattern Args: img: an image to be perturbed towards the desired activation pattern targetedNeuronIndex: Indices of neuron with the goal of satisfying the pattern desiredNAP: desired shape (>0 is set to 1; <= 0 is set to -1) model: neural network under analysis """ steps = 20000 loss = nn.L1Loss() alpha = 0.1 x = Variable(img, requires_grad=True) result = img delta = torch.zeros(result.shape) for step in range(steps): if (step == steps -1): print("Unable to find an image to satisfy the required pattern!") zero_gradients(x) out, intermediate = model.forwardWithIntermediate(x) # The output label is first set to values equal to the computed layer. labels = torch.tensor(intermediate) # Modify the label, such that if the sign of the specified neuron is different, use the desired value. By doing so, # (1) the loss of other un-specified neuron will be 0, # (2) for specified neuron with correct sign, the loss will be 0 # (3) for specified neuron without correct sign, there will be a loss satisfied = True for i in range(len(targetedNeuronIndex)): if not ((intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[i]]>0) == (desiredNAP[i]>0)): satisfied = False labels[0][targetedNeuronIndex[i]] = desiredNAP[i] if satisfied: if step == 0: print("The original image already satisfy the required pattern!") return result.cpu(), True else : print("Found an image to successfully create the required pattern:") return result.cpu(), True if(step == 0): print(intermediate) print(labels) y = Variable(labels) _loss = loss(intermediate, y) _loss.backward() result = x.data - (alpha * x.grad.data) # TODO: Here we are not clamping the input result = torch.clamp(result, 0.0, 1.0) x.data = result # x.data = x.data - (alpha * x.grad.data) if(step % 500 == 0): print(str(step)+": "+str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[0]]) + ", "+ str(intermediate.detach().numpy().squeeze(0)[targetedNeuronIndex[1]])) return result.cpu(), False
def forward(self, x_acc, x_mic, acc_lens, mic_lens): # (int(np.shape(x_acc)[0] / 5) is the current batch size acc_s_batch = x_acc[0 : np.shape(x_acc)[0] : 5, :, :] mic_s_batch = x_mic[0 : np.shape(x_mic)[0] : 5, :, :] acc_s_lengths = torch.tensor(acc_lens[0 : np.shape(x_acc)[0] : 5]) mic_s_lengths = torch.tensor(mic_lens[0 : np.shape(x_acc)[0] : 5]) acc_s_batch = acc_s_batch.permute(0, 2, 1) mic_s_batch = mic_s_batch.permute(0, 2, 1) acc_s_batch = acc_s_batch.to(self.device) mic_s_batch = mic_s_batch.to(self.device) out_acc_s = self.tcn_subject_acc(acc_s_batch.float()) out_mic_s = self.tcn_subject_mic(mic_s_batch.float()) ending_acc_s_outputs = torch.zeros( (int(np.shape(x_acc)[0] / 5), self.hidden_dim) ) ending_mic_s_outputs = torch.zeros( (int(np.shape(x_acc)[0] / 5), self.hidden_dim) ) for b_num, cur_size in zip(np.arange(int(np.shape(x_acc)[0] / 5)), acc_s_lengths): if cur_size == 0: continue else: ending_acc_s_outputs[b_num, :] = out_acc_s[b_num, :, cur_size - 1] for b_num, cur_size in zip(np.arange(int(np.shape(x_acc)[0] / 5)), mic_s_lengths): if cur_size == 0: continue else: ending_mic_s_outputs[b_num, :] = out_mic_s[b_num, :, cur_size - 1] # Get last time step out_acc_s_last_timestep = ending_acc_s_outputs.to(self.device) out_mic_s_last_timestep = ending_mic_s_outputs.to(self.device) out_neighbors_acc = [] out_neighbors_mic = [] # Neighbor's data for i in np.arange(1, 5): cur_acc_n_batch = x_acc[i : np.shape(x_acc)[0] : 5, :, :] cur_mic_n_batch = x_mic[i : np.shape(x_mic)[0] : 5, :, :] cur_acc_n_lengths = torch.tensor(acc_lens[i : np.shape(x_acc)[0] : 5]) cur_mic_n_lengths = torch.tensor(mic_lens[i : np.shape(x_mic)[0] : 5]) cur_acc_n_batch = cur_acc_n_batch.permute(0, 2, 1) cur_mic_n_batch = cur_mic_n_batch.permute(0, 2, 1) cur_acc_n_batch = cur_acc_n_batch.to(self.device) cur_mic_n_batch = cur_mic_n_batch.to(self.device) out_neighbor_acc = self.tcn_neighbors_acc(cur_acc_n_batch.float()) out_neighbor_mic = self.tcn_neighbors_mic(cur_mic_n_batch.float()) ending_acc_n_outputs = torch.zeros( (int(np.shape(x_acc)[0] / 5), self.hidden_dim) ) ending_mic_n_outputs = torch.zeros( (int(np.shape(x_mic)[0] / 5), self.hidden_dim) ) for b_num, cur_size in zip( np.arange(int(np.shape(x_acc)[0] / 5)), cur_acc_n_lengths ): if cur_size == 0: continue else: ending_acc_n_outputs[b_num, :] = out_neighbor_acc[ b_num, :, cur_size - 1 ] for b_num, cur_size in zip( np.arange(int(np.shape(x_acc)[0] / 5)), cur_mic_n_lengths ): if cur_size == 0: continue else: ending_mic_n_outputs[b_num, :] = out_neighbor_mic[ b_num, :, cur_size - 1 ] ending_acc_n_outputs = ending_acc_n_outputs.to(self.device) ending_mic_n_outputs = ending_mic_n_outputs.to(self.device) out_neighbors_acc.append(ending_acc_n_outputs) out_neighbors_mic.append(ending_mic_n_outputs) out_neighbors_acc.insert(0, out_acc_s_last_timestep) out_neighbors_mic.insert(0, out_mic_s_last_timestep) pooled_outputs_acc = [] pooled_outputs_mic = [] for bn in np.arange(int(np.shape(x_acc)[0] / 5)): # Get the data of the subject entered the ESM sbj_acc = out_neighbors_acc[0][bn, :].view(1, self.hidden_dim) sbj_mic = out_neighbors_mic[0][bn, :].view(1, self.hidden_dim) # Get the neighbours n1_acc = out_neighbors_acc[1][bn, :].view(1, self.hidden_dim) n2_acc = out_neighbors_acc[2][bn, :].view(1, self.hidden_dim) n3_acc = out_neighbors_acc[3][bn, :].view(1, self.hidden_dim) n4_acc = out_neighbors_acc[4][bn, :].view(1, self.hidden_dim) n1_mic = out_neighbors_mic[1][bn, :].view(1, self.hidden_dim) n2_mic = out_neighbors_mic[2][bn, :].view(1, self.hidden_dim) n3_mic = out_neighbors_mic[3][bn, :].view(1, self.hidden_dim) n4_mic = out_neighbors_mic[4][bn, :].view(1, self.hidden_dim) if len(torch.nonzero(n1_acc)) != 0: sbj_acc = torch.cat((sbj_acc, n1_acc), dim=0) if len(torch.nonzero(n2_acc)) != 0: sbj_acc = torch.cat((sbj_acc, n2_acc), dim=0) if len(torch.nonzero(n3_acc)) != 0: sbj_acc = torch.cat((sbj_acc, n3_acc), dim=0) if len(torch.nonzero(n4_acc)) != 0: sbj_acc = torch.cat((sbj_acc, n4_acc), dim=0) if len(torch.nonzero(n1_mic)) != 0: sbj_mic = torch.cat((sbj_mic, n1_mic), dim=0) if len(torch.nonzero(n2_mic)) != 0: sbj_mic = torch.cat((sbj_mic, n2_mic), dim=0) if len(torch.nonzero(n3_mic)) != 0: sbj_mic = torch.cat((sbj_mic, n3_mic), dim=0) if len(torch.nonzero(n4_mic)) != 0: sbj_mic = torch.cat((sbj_mic, n4_mic), dim=0) avg_pooled_acc = torch.mean(sbj_acc, dim=0) pooled_outputs_acc.append(avg_pooled_acc) avg_pooled_mic = torch.mean(sbj_mic, dim=0) pooled_outputs_mic.append(avg_pooled_mic) pooled_outputs_acc = torch.stack(pooled_outputs_acc) pooled_outputs_mic = torch.stack(pooled_outputs_mic) all_out = torch.cat((pooled_outputs_acc, pooled_outputs_mic), 1) all_out_eff = self.drp_eff(F.leaky_relu(self.fc_eff(all_out))) all_out_fst = self.drp_fst(F.leaky_relu(self.fc_fst(all_out))) all_out_sts = self.drp_sts(F.leaky_relu(self.fc_sts(all_out))) out_eff = self.out_eff(all_out_eff) out_fst = self.out_fst(all_out_fst) out_sts = self.out_sts(all_out_sts) return out_eff, out_fst, out_sts
def evaluate(args, model: PreTrainedModel, tokenizer: PreTrainedTokenizer, prefix="") -> Dict: # Loop to handle MNLI double evaluation (matched, mis-matched) eval_output_dir = args.output_dir eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True) if args.local_rank in [-1, 0]: os.makedirs(eval_output_dir, exist_ok=True) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly def collate(examples: List[torch.Tensor]): if tokenizer._pad_token is None: return pad_sequence(examples, batch_first=True) return pad_sequence(examples, batch_first=True, padding_value=tokenizer.pad_token_id) eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, collate_fn=collate) # multi-gpu evaluate if args.n_gpu > 1: model = torch.nn.DataParallel(model) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) logger.info(" Batch size = %d", args.eval_batch_size) eval_loss = 0.0 nb_eval_steps = 0 model.eval() for batch in tqdm(eval_dataloader, desc="Evaluating"): inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch) inputs = inputs.to(args.device) labels = labels.to(args.device) with torch.no_grad(): outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model( inputs, labels=labels) lm_loss = outputs[0] eval_loss += lm_loss.mean().item() nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps perplexity = torch.exp(torch.tensor(eval_loss)) result = {"perplexity": perplexity} output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") with open(output_eval_file, "w") as writer: logger.info("***** Eval results {} *****".format(prefix)) for key in sorted(result.keys()): logger.info(" %s = %s", key, str(result[key])) writer.write("%s = %s\n" % (key, str(result[key]))) return result
def reference(ref_data): torch_test_data = torch.tensor(input_data, requires_grad=False) torch_softshrink = torch.nn.Softshrink(lambd=1.5) m = torch_softshrink(torch_test_data) return [m]
def visualize_output( pdf: t.Tensor, ex: batched_example.BatchedExample, task_type: configuration.TaskType, batch_indices: Iterable[int] = None) -> List[List[t.Tensor]]: """Visualizes the predicted outputs next to the ground-truth. Args: pdf: The class probabilities output by the model. ex: The input batch, containing the ground-truth among others. task_type: The task type. batch_indices: The batch elements to visualized. Visualizes all by default. Returns: One list of images for each batch element, containing the element visualized from different angles. """ palette = misc_util.to_tensor(colors.DEFAULT_COLOR_PALETTE, t.float32) palette = palette.to(pdf.device) output_images = [] scene_num_tri = [n.sum().item() for n in ex.mesh_num_tri] offsets = t.tensor(scene_num_tri).cumsum(0).tolist() offsets = [0] + offsets[:-1] if not batch_indices: batch_indices = range(pdf.shape[0]) vis = visualization_artifacts pred_lbl, gt_lbl = extract_labels(pdf, ex, task_type) for batch_idx in batch_indices: v2x = ex.v2x_transform[batch_idx].inverse() gt_mesh_labels = ex.mesh_labels[batch_idx] artifacts_3d = [] # Marching cubes visualization for the predicted volume if task_type == configuration.TaskType.FG_BG: assert gt_mesh_labels.shape == (1, ) mc_colors = palette.new_tensor([0, gt_mesh_labels[0]], dtype=t.int64) mc_colors = palette[mc_colors] else: num_classes = pdf.shape[1] mc_colors = palette[:num_classes] artifacts_3d.append( vis.MarchingCubesArtifact(pdf[batch_idx], v2x, mc_colors)) # Mesh visualization of the GT scene gt_mesh_colors = palette[gt_mesh_labels.to(t.int64)] mesh_num_tri = ex.mesh_num_tri[batch_idx] offset = offsets[batch_idx] gt_vertices = ex.vertices[offset:offset + scene_num_tri[batch_idx]] artifacts_3d.append( vis.MultiMeshArtifact(gt_vertices, mesh_num_tri, mesh_colors=gt_mesh_colors)) # Voxel grid visualizations of both the predicted volume and the GT artifacts_3d.append(vis.VoxelGridArtifact(pred_lbl[batch_idx], v2x)) artifacts_3d.append(vis.VoxelGridArtifact(gt_lbl[batch_idx], v2x)) artifacts = [ vis.ImageArtifact(ex.input_image[batch_idx]), artifacts_3d ] camera_images = vis.visualize_artifacts(artifacts, ex.camera_transform[batch_idx], ex.view_transform[batch_idx]) output_images.append(camera_images) return output_images
def __init__(self): super(Norm, self).__init__() self.gamma = Parameter(torch.tensor(1.0)) self.beta = Parameter(torch.tensor(0.0))
def _calculate_per_channel_qparams(self, min_vals, max_vals): # type: (Tensor, Tensor) -> Tuple[Tensor, Tensor] r"""Calculates the per channel quantization parameters, given min and max value tensors. Args: min_vals: Minimum values per channel max_vals: Maximum values per channel Returns: scales: Per channel scales tensor of shape (#channels,) zero_points: Per channel zero points tensor of shape (#channels,) """ if min_vals.numel() == 0 or max_vals.numel() == 0: warnings.warn("must run observer before calling calculate_qparams.\ Returning default scale and zero point ") return torch.tensor([1.0]), torch.tensor([0]) diff = min_vals <= max_vals assert (torch.sum(diff) == len(diff) ), "min_vals should be less than max_vals for indices." scales = torch.empty(min_vals.size(), dtype=torch.float32) zero_points = torch.empty(min_vals.size(), dtype=torch.int64) if self.dtype == torch.qint8: if self.reduce_range: qmin, qmax = -64, 63 else: qmin, qmax = -128, 127 else: if self.reduce_range: qmin, qmax = 0, 127 else: qmin, qmax = 0, 255 max_vals, min_vals = max_vals.to(dtype=torch.float), min_vals.to( dtype=torch.float) min_vals = torch.min( min_vals, torch.tensor([0.], device=min_vals.device, dtype=torch.float)) max_vals = torch.max( max_vals, torch.tensor([0.], device=max_vals.device, dtype=torch.float)) if torch.equal(max_vals, min_vals): scales.fill_(1.0) zero_points.fill_(0) else: if self.qscheme == torch.per_tensor_symmetric or self.qscheme == torch.per_channel_symmetric: max_vals = torch.max(-min_vals, max_vals) scales = max_vals / ((qmax - qmin) / 2) scales = torch.max( scales, torch.tensor([self.eps], device=scales.device, dtype=scales.dtype)) if self.dtype == torch.qint8: zp = 0 else: zp = 128 zero_points.fill_(zp) else: scales = (max_vals - min_vals) / float(qmax - qmin) scales = torch.max( scales, torch.tensor([self.eps], device=scales.device)) zero_points = qmin - torch.round(min_vals / scales) zero_points = torch.max( zero_points, torch.tensor([qmin], dtype=zero_points.dtype, device=zero_points.device)) zero_points = torch.min( zero_points, torch.tensor([qmax], dtype=zero_points.dtype, device=zero_points.device)) zero_points = zero_points.to(dtype=torch.int64) scales.to(dtype=torch.float) return scales, zero_points
def evaluate(args, model, processor, tokenizer, global_step, input_dir, prefix=""): retrievers = dict() for kg in args.use_kgs: logger.info("Initialize kg:{}".format(kg)) kg_path = os.path.join(input_dir, args.kg_paths[kg]) data_path = os.path.join(args.data_dir, args.kg_paths[kg]) if not os.path.exists(kg_path): logger.warning("need prepare training dataset firstly, program exit") exit() retrievers[kg] = initialize_kg_retriever(kg, kg_path, data_path, args.cache_file_suffix) dataset, examples_tokenized, features, wn_synset_graphs, wn_synset_graphs_label_dict = \ load_and_cache_examples(args, processor, retrievers, relation_list=args.relation_list, input_dir=input_dir, evaluate=True, output_examples=True) if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.mkdir(args.output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset, shuffle=False) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # synset_graphs_batch = [] # for batch_index in eval_dataloader.batch_sampler: # synset_graphs_batch.append([i for i in batch_index]) # multi-gpu evaluate if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) if args.local_rank != -1 and not isinstance(model, torch.nn.parallel.DistributedDataParallel): model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True ) # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Dataset size = %d", len(dataset)) logger.info(" Batch size = %d", args.eval_batch_size) if args.local_rank == -1: logger.warning("program exits and please use pytorch DDP framework") exit() else: # all_results = [] # all_start_logits = torch.tensor([], dtype=torch.float32, device=args.device) # all_end_logits = torch.tensor([], dtype=torch.float32, device=args.device) # all_unique_ids = [] all_pred = torch.tensor([], dtype=torch.long, device=args.device) all_label_ids = torch.tensor([], dtype=torch.long, device=args.device) all_question_ids = torch.tensor([], dtype=torch.long, device=args.device) # start_time = timeit.default_timer() epoch_iterator = tqdm(eval_dataloader, desc="Evaluating Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): model.eval() batch = tuple(t.to(args.device) for t in batch) batch_synset_graphs = batch[3] with torch.no_grad(): inputs = create_input(args, batch, global_step, batch_synset_graphs=batch_synset_graphs, wn_synset_graphs=wn_synset_graphs, evaluate=True) feature_indices = batch[3] outputs = model(**inputs) logits, label_ids, qas_ids = outputs[1], outputs[2], outputs[3] all_pred = torch.cat((all_pred, torch.argmax(logits, axis=-1)), dim=0) all_label_ids = torch.cat((all_label_ids, label_ids), dim=0) all_question_ids = torch.cat((all_question_ids, qas_ids), dim=0) start_time = timeit.default_timer() all_pred_list = [torch.zeros_like(all_pred, device=args.device) for _ in range(torch.distributed.get_world_size())] all_label_ids_list = [torch.zeros_like(all_label_ids, device=args.device) for _ in range(torch.distributed.get_world_size())] all_question_ids_list = [torch.zeros_like(all_question_ids, device=args.device) for _ in range(torch.distributed.get_world_size())] all_gather(all_pred_list, all_pred) all_gather(all_label_ids_list, all_label_ids) all_gather(all_question_ids_list, all_question_ids) logger.info( "time for gather communication:{} in rank {}".format(timeit.default_timer() - start_time, args.local_rank)) if args.local_rank == 0: start_time = timeit.default_timer() all_results = [] all_pred_list = all_pred_list all_label_ids_list = all_label_ids_list all_question_ids_list = all_question_ids_list preds = np.asarray([], dtype=int) label_values = np.asarray([], dtype=int) question_ids = np.asarray([], dtype=int) for batch_idx, batch_preds in enumerate(all_pred_list): preds = np.concatenate((preds, batch_preds.cpu().detach().numpy()), axis=0) label_values = np.concatenate((label_values, all_label_ids_list[batch_idx].cpu().detach().numpy()), axis=0) question_ids = np.concatenate((question_ids, all_question_ids_list[batch_idx].cpu().detach().numpy()), axis=0) if not args.test: acc = float((preds == label_values).mean()) results = {'acc': acc} else: results = None if args.write_preds: guids = [] for f in features: guids.append(f.guid[0]) guids = np.asarray(guids, dtype='<U18') assert len(preds)==len(guids) write_prediction(preds, guids, "copa", prefix) return results else: return None
def main(): parser = argparse.ArgumentParser() ## Required parameters parser.add_argument("--bert_model", default=None, type=str, required=True, help="Bert pre-trained model selected in the list: bert-base-uncased, " "bert-large-uncased, bert-base-cased, bert-base-multilingual, bert-base-chinese.") parser.add_argument("--output_dir", default=None, type=str, required=True, help="The output directory where the model checkpoints will be written.") ## Other parameters parser.add_argument("--train_file", default=None, type=str, help="SQuAD json for training. E.g., train-v1.1.json") parser.add_argument("--predict_file", default=None, type=str, help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") parser.add_argument("--max_seq_length", default=384, type=int, help="The maximum total input sequence length after WordPiece tokenization. Sequences " "longer than this will be truncated, and sequences shorter than this will be padded.") parser.add_argument("--doc_stride", default=128, type=int, help="When splitting up a long document into chunks, how much stride to take between chunks.") parser.add_argument("--max_query_length", default=64, type=int, help="The maximum number of tokens for the question. Questions longer than this will " "be truncated to this length.") parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.") parser.add_argument("--do_predict", default=False, action='store_true', help="Whether to run eval on the dev set.") parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.") parser.add_argument("--predict_batch_size", default=8, type=int, help="Total batch size for predictions.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--num_train_epochs", default=3.0, type=float, help="Total number of training epochs to perform.") parser.add_argument("--warmup_proportion", default=0.1, type=float, help="Proportion of training to perform linear learning rate warmup for. E.g., 0.1 = 10% " "of training.") parser.add_argument("--n_best_size", default=20, type=int, help="The total number of n-best predictions to generate in the nbest_predictions.json " "output file.") parser.add_argument("--max_answer_length", default=30, type=int, help="The maximum length of an answer that can be generated. This is needed because the start " "and end predictions are not conditioned on one another.") parser.add_argument("--verbose_logging", default=False, action='store_true', help="If true, all of the warnings related to data processing will be printed. " "A number of warnings are expected for a normal SQuAD evaluation.") parser.add_argument("--no_cuda", default=False, action='store_true', help="Whether not to use CUDA when available") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--local_rank", type=int, default=-1, help="local_rank for distributed training on gpus") parser.add_argument('--optimize_on_cpu', default=False, action='store_true', help="Whether to perform optimization and keep the optimizer averages on CPU") parser.add_argument('--fp16', default=False, action='store_true', help="Whether to use 16-bit float precision instead of 32-bit") parser.add_argument('--loss_scale', type=float, default=128, help='Loss scaling, positive power of 2 values can improve fp16 convergence.') parser.add_argument('--do_lower_case', action="store_true", default=True, help="Lowercase the input") args = parser.parse_args() if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") n_gpu = torch.cuda.device_count() else: device = torch.device("cuda", args.local_rank) n_gpu = 1 # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.distributed.init_process_group(backend='nccl') if args.fp16: logger.info("16-bits training currently not supported in distributed training") args.fp16 = False # (see https://github.com/pytorch/pytorch/pull/13496) logger.info("device: {} n_gpu: {}, distributed training: {}, 16-bits trainiing: {}".format( device, n_gpu, bool(args.local_rank != -1), args.fp16)) if args.gradient_accumulation_steps < 1: raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format( args.gradient_accumulation_steps)) args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) if n_gpu > 0: torch.cuda.manual_seed_all(args.seed) if not args.do_train and not args.do_predict: raise ValueError("At least one of `do_train` or `do_predict` must be True.") if args.do_train: if not args.train_file: raise ValueError( "If `do_train` is True, then `train_file` must be specified.") if args.do_predict: if not args.predict_file: raise ValueError( "If `do_predict` is True, then `predict_file` must be specified.") if os.path.exists(args.output_dir) and os.listdir(args.output_dir): raise ValueError("Output directory () already exists and is not empty.") os.makedirs(args.output_dir, exist_ok=True) tokenizer = BertTokenizer.from_pretrained(args.bert_model) train_examples = None num_train_steps = None if args.do_train: train_examples = read_squad_examples( input_file=args.train_file, is_training=True) num_train_steps = int( len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs) # Prepare model model = BertForQuestionAnswering.from_pretrained(args.bert_model) if args.fp16: model.half() model.to(device) if args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank) elif n_gpu > 1: model = torch.nn.DataParallel(model) # Prepare optimizer if args.fp16: param_optimizer = [(n, param.clone().detach().to('cpu').float().requires_grad_()) \ for n, param in model.named_parameters()] elif args.optimize_on_cpu: param_optimizer = [(n, param.clone().detach().to('cpu').requires_grad_()) \ for n, param in model.named_parameters()] else: param_optimizer = list(model.named_parameters()) no_decay = ['bias', 'gamma', 'beta'] optimizer_grouped_parameters = [ {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0} ] optimizer = BertAdam(optimizer_grouped_parameters, lr=args.learning_rate, warmup=args.warmup_proportion, t_total=num_train_steps) global_step = 0 if args.do_train: train_features = convert_examples_to_features( examples=train_examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=True) logger.info("***** Running training *****") logger.info(" Num orig examples = %d", len(train_examples)) logger.info(" Num split examples = %d", len(train_features)) logger.info(" Batch size = %d", args.train_batch_size) logger.info(" Num steps = %d", num_train_steps) all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long) all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long) all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long) train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_start_positions, all_end_positions) if args.local_rank == -1: train_sampler = RandomSampler(train_data) else: train_sampler = DistributedSampler(train_data) train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size) model.train() for _ in trange(int(args.num_train_epochs), desc="Epoch"): ep = 0 for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")): if n_gpu == 1: batch = tuple(t.to(device) for t in batch) # multi-gpu does scattering it-self input_ids, input_mask, segment_ids, start_positions, end_positions = batch loss = model(input_ids, segment_ids, input_mask, start_positions, end_positions) if n_gpu > 1: loss = loss.mean() # mean() to average on multi-gpu. if args.fp16 and args.loss_scale != 1.0: # rescale loss for fp16 training # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html loss = loss * args.loss_scale if args.gradient_accumulation_steps > 1: loss = loss / args.gradient_accumulation_steps loss.backward() if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16 or args.optimize_on_cpu: if args.fp16 and args.loss_scale != 1.0: # scale down gradients for fp16 training for param in model.parameters(): if param.grad is not None: param.grad.data = param.grad.data / args.loss_scale is_nan = set_optimizer_params_grad(param_optimizer, model.named_parameters(), test_nan=True) if is_nan: logger.info("FP16 TRAINING: Nan in gradients, reducing loss scaling") args.loss_scale = args.loss_scale / 2 model.zero_grad() continue optimizer.step() copy_optimizer_params_to_model(model.named_parameters(), param_optimizer) else: optimizer.step() model.zero_grad() global_step += 1 torch.save(model.state_dict(), (args.output_dir+ "train_epoch" + ep + ".json")) ep = ep +1 if args.do_predict: eval_examples = read_squad_examples( input_file=args.predict_file, is_training=False) eval_features = convert_examples_to_features( examples=eval_examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=False) logger.info("***** Running predictions *****") logger.info(" Num orig examples = %d", len(eval_examples)) logger.info(" Num split examples = %d", len(eval_features)) logger.info(" Batch size = %d", args.predict_batch_size) all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_example_index) if args.local_rank == -1: eval_sampler = SequentialSampler(eval_data) else: eval_sampler = DistributedSampler(eval_data) eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.predict_batch_size) model.eval() all_results = [] logger.info("Start evaluating") for input_ids, input_mask, segment_ids, example_indices in tqdm(eval_dataloader, desc="Evaluating"): if len(all_results) % 1000 == 0: logger.info("Processing example: %d" % (len(all_results))) input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) with torch.no_grad(): batch_start_logits, batch_end_logits = model(input_ids, segment_ids, input_mask) for i, example_index in enumerate(example_indices): start_logits = batch_start_logits[i].detach().cpu().tolist() end_logits = batch_end_logits[i].detach().cpu().tolist() eval_feature = eval_features[example_index.item()] unique_id = int(eval_feature.unique_id) all_results.append(RawResult(unique_id=unique_id, start_logits=start_logits, end_logits=end_logits)) output_prediction_file = os.path.join(args.output_dir, "predictions.json") output_nbest_file = os.path.join(args.output_dir, "nbest_predictions.json") write_predictions(eval_examples, eval_features, all_results, args.n_best_size, args.max_answer_length, args.do_lower_case, output_prediction_file, output_nbest_file, args.verbose_logging)
def forward(self, im_data, im_info, gt_boxes, num_boxes, support_ims, all_cls_gt_boxes): im_batch = im_data im_info = im_info.data gt_boxes = gt_boxes.data num_boxes = num_boxes.data support_ims = support_ims.data batch_size = im_batch.size(0) x = self.resnet_base(im_batch) pos_sup_ims = support_ims[:, :self.num_shot, :, :, :].contiguous() pos_sup_feats = self.resnet_base(pos_sup_ims.view( batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80] if self.training: neg_sup_ims = support_ims[:, self.num_shot:, :, :, :].contiguous() neg_sup_feats = self.resnet_base(neg_sup_ims.view( batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80] x1 = self.layer1(x) x2 = self.layer2(x1) x3 = self.layer3(x2) x4 = self.layer4(x3) pos_s1 = self.layer1(pos_sup_feats) pos_s2 = self.layer2(pos_s1) pos_s3 = self.layer3(pos_s2) pos_s4 = self.layer4(pos_s3) if self.training: neg_s1 = self.layer1(neg_sup_feats) neg_s2 = self.layer2(neg_s1) neg_s3 = self.layer3(neg_s2) neg_s4 = self.layer4(neg_s3) features = self.fpn([x2, x3, x4])[1:] pos_s_features = self.fpn([pos_s2, pos_s3, pos_s4])[1:] if self.training: neg_s_features = self.fpn([neg_s2, neg_s3, neg_s4])[1:] correlation_features = self.attention_module(features, pos_s_features) if self.training: neg_correlation_features = self.attention_module(features, neg_s_features) regression = torch.cat([self.regressionModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, 4] classification = torch.cat([self.classificationModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, n_class] if self.training: neg_regression = torch.cat([self.regressionModel(feat) for feat in neg_correlation_features], dim=1) neg_classification = torch.cat([self.classificationModel(feat) for feat in neg_correlation_features], dim=1) anchors = self.anchors(im_batch) rpn_loss_cls = torch.zeros(1).cuda() rpn_loss_bbox = torch.zeros(1).cuda() RCNN_loss_cls = torch.zeros(1).cuda() RCNN_loss_bbox = torch.zeros(1).cuda() rois_label = torch.zeros(10).cuda() rois = None cls_prob = None bbox_pred = None if self.training: pos_RCNN_loss_cls, RCNN_loss_bbox = self.focalLoss(classification, regression, anchors, gt_boxes) empty_gt_boxes = torch.zeros_like(gt_boxes) - 1. neg_RCNN_loss_cls, _ = self.focalLoss(neg_classification, neg_regression, anchors, empty_gt_boxes) RCNN_loss_cls = (pos_RCNN_loss_cls + neg_RCNN_loss_cls) / 2 return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label else: transformed_anchors = self.regressBoxes(anchors, regression) transformed_anchors = self.clipBoxes(transformed_anchors, im_batch) finalResult = [[], [], []] finalScores = torch.Tensor([]) finalAnchorBoxesIndexes = torch.Tensor([]).long() finalAnchorBoxesCoordinates = torch.Tensor([]) if torch.cuda.is_available(): finalScores = finalScores.cuda() finalAnchorBoxesIndexes = finalAnchorBoxesIndexes.cuda() finalAnchorBoxesCoordinates = finalAnchorBoxesCoordinates.cuda() # scores = torch.squeeze(classification[:, :, 1]) scores = F.softmax(classification[0], 1)[:, 1] scores_over_thresh = (scores > 0.5) if scores_over_thresh.sum() == 0: return None, None, np.array([0., 0., 0., 0.]) scores = scores[scores_over_thresh] anchorBoxes = torch.squeeze(transformed_anchors) anchorBoxes = anchorBoxes[scores_over_thresh] anchors_nms_idx = nms(anchorBoxes, scores, 0.5) finalScores = torch.cat((finalScores, scores[anchors_nms_idx])) # [n_predict] finalAnchorBoxesIndexesValue = torch.tensor([1] * anchors_nms_idx.shape[0]) if torch.cuda.is_available(): finalAnchorBoxesIndexesValue = finalAnchorBoxesIndexesValue.cuda() finalAnchorBoxesIndexes = torch.cat((finalAnchorBoxesIndexes, finalAnchorBoxesIndexesValue)) # [n_predict] finalAnchorBoxesCoordinates = torch.cat((finalAnchorBoxesCoordinates, anchorBoxes[anchors_nms_idx])) # [n_predict, 4] return [finalScores, finalAnchorBoxesIndexes, finalAnchorBoxesCoordinates]
def fit_single_frame(img, keypoints, body_model, camera, joint_weights, body_pose_prior, jaw_prior, left_hand_prior, right_hand_prior, shape_prior, expr_prior, angle_prior, result_fn='out.pkl', mesh_fn='out.obj', out_img_fn='overlay.png', loss_type='smplify', use_cuda=True, init_joints_idxs=(9, 12, 2, 5), use_face=True, use_hands=True, data_weights=None, body_pose_prior_weights=None, hand_pose_prior_weights=None, jaw_pose_prior_weights=None, shape_weights=None, expr_weights=None, hand_joints_weights=None, face_joints_weights=None, depth_loss_weight=1e2, interpenetration=True, coll_loss_weights=None, df_cone_height=0.5, penalize_outside=True, max_collisions=8, point2plane=False, part_segm_fn='', focal_length=5000., side_view_thsh=25., rho=100, vposer_latent_dim=32, vposer_ckpt='', use_joints_conf=False, interactive=True, visualize=True, save_meshes=True, degrees=None, batch_size=1, dtype=torch.float32, ign_part_pairs=None, left_shoulder_idx=2, right_shoulder_idx=5, **kwargs): assert batch_size == 1, 'PyTorch L-BFGS only supports batch_size == 1' device = torch.device('cuda') if use_cuda else torch.device('cpu') if degrees is None: degrees = [0, 90, 180, 270] if data_weights is None: data_weights = [1, ] * 5 if body_pose_prior_weights is None: body_pose_prior_weights = [4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78] msg = ( 'Number of Body pose prior weights {}'.format( len(body_pose_prior_weights)) + ' does not match the number of data term weights {}'.format( len(data_weights))) assert (len(data_weights) == len(body_pose_prior_weights)), msg if use_hands: if hand_pose_prior_weights is None: hand_pose_prior_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1] msg = ('Number of Body pose prior weights does not match the' + ' number of hand pose prior weights') assert (len(hand_pose_prior_weights) == len(body_pose_prior_weights)), msg if hand_joints_weights is None: hand_joints_weights = [0.0, 0.0, 0.0, 1.0] msg = ('Number of Body pose prior weights does not match the' + ' number of hand joint distance weights') assert (len(hand_joints_weights) == len(body_pose_prior_weights)), msg if shape_weights is None: shape_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1] msg = ('Number of Body pose prior weights = {} does not match the' + ' number of Shape prior weights = {}') assert (len(shape_weights) == len(body_pose_prior_weights)), msg.format( len(shape_weights), len(body_pose_prior_weights)) if use_face: if jaw_pose_prior_weights is None: jaw_pose_prior_weights = [[x] * 3 for x in shape_weights] else: jaw_pose_prior_weights = map(lambda x: map(float, x.split(',')), jaw_pose_prior_weights) jaw_pose_prior_weights = [list(w) for w in jaw_pose_prior_weights] msg = ('Number of Body pose prior weights does not match the' + ' number of jaw pose prior weights') assert (len(jaw_pose_prior_weights) == len(body_pose_prior_weights)), msg if expr_weights is None: expr_weights = [1e2, 5 * 1e1, 1e1, .5 * 1e1] msg = ('Number of Body pose prior weights = {} does not match the' + ' number of Expression prior weights = {}') assert (len(expr_weights) == len(body_pose_prior_weights)), msg.format( len(body_pose_prior_weights), len(expr_weights)) if face_joints_weights is None: face_joints_weights = [0.0, 0.0, 0.0, 1.0] msg = ('Number of Body pose prior weights does not match the' + ' number of face joint distance weights') assert (len(face_joints_weights) == len(body_pose_prior_weights)), msg if coll_loss_weights is None: coll_loss_weights = [0.0] * len(body_pose_prior_weights) msg = ('Number of Body pose prior weights does not match the' + ' number of collision loss weights') assert (len(coll_loss_weights) == len(body_pose_prior_weights)), msg use_vposer = kwargs.get('use_vposer', True) vposer, pose_embedding = [None, ] * 2 if use_vposer: pose_embedding = torch.zeros([batch_size, 32], dtype=dtype, device=device, requires_grad=True) vposer_ckpt = osp.expandvars(vposer_ckpt) vposer, _ = load_vposer(vposer_ckpt, vp_model='snapshot') vposer = vposer.to(device=device) vposer.eval() if use_vposer: body_mean_pose = torch.zeros([batch_size, vposer_latent_dim], dtype=dtype) else: body_mean_pose = body_pose_prior.get_mean().detach().cpu() keypoint_data = torch.tensor(keypoints, dtype=dtype) gt_joints = keypoint_data[:, :, :2] if use_joints_conf: joints_conf = keypoint_data[:, :, 2].reshape(1, -1) # Transfer the data to the correct device gt_joints = gt_joints.to(device=device, dtype=dtype) if use_joints_conf: joints_conf = joints_conf.to(device=device, dtype=dtype) # Create the search tree search_tree = None pen_distance = None filter_faces = None if interpenetration: from mesh_intersection.bvh_search_tree import BVH import mesh_intersection.loss as collisions_loss from mesh_intersection.filter_faces import FilterFaces assert use_cuda, 'Interpenetration term can only be used with CUDA' assert torch.cuda.is_available(), \ 'No CUDA Device! Interpenetration term can only be used' + \ ' with CUDA' search_tree = BVH(max_collisions=max_collisions) pen_distance = \ collisions_loss.DistanceFieldPenetrationLoss( sigma=df_cone_height, point2plane=point2plane, vectorized=True, penalize_outside=penalize_outside) if part_segm_fn: # Read the part segmentation part_segm_fn = os.path.expandvars(part_segm_fn) with open(part_segm_fn, 'rb') as faces_parents_file: face_segm_data = pickle.load(faces_parents_file, encoding='latin1') faces_segm = face_segm_data['segm'] faces_parents = face_segm_data['parents'] # Create the module used to filter invalid collision pairs filter_faces = FilterFaces( faces_segm=faces_segm, faces_parents=faces_parents, ign_part_pairs=ign_part_pairs).to(device=device) # Weights used for the pose prior and the shape prior opt_weights_dict = {'data_weight': data_weights, 'body_pose_weight': body_pose_prior_weights, 'shape_weight': shape_weights} if use_face: opt_weights_dict['face_weight'] = face_joints_weights opt_weights_dict['expr_prior_weight'] = expr_weights opt_weights_dict['jaw_prior_weight'] = jaw_pose_prior_weights if use_hands: opt_weights_dict['hand_weight'] = hand_joints_weights opt_weights_dict['hand_prior_weight'] = hand_pose_prior_weights if interpenetration: opt_weights_dict['coll_loss_weight'] = coll_loss_weights keys = opt_weights_dict.keys() opt_weights = [dict(zip(keys, vals)) for vals in zip(*(opt_weights_dict[k] for k in keys if opt_weights_dict[k] is not None))] for weight_list in opt_weights: for key in weight_list: weight_list[key] = torch.tensor(weight_list[key], device=device, dtype=dtype) # The indices of the joints used for the initialization of the camera init_joints_idxs = torch.tensor(init_joints_idxs, device=device) edge_indices = kwargs.get('body_tri_idxs') init_t = fitting.guess_init(body_model, gt_joints, edge_indices, use_vposer=use_vposer, vposer=vposer, pose_embedding=pose_embedding, model_type=kwargs.get('model_type', 'smpl'), focal_length=focal_length, dtype=dtype) camera_loss = fitting.create_loss('camera_init', trans_estimation=init_t, init_joints_idxs=init_joints_idxs, depth_loss_weight=depth_loss_weight, dtype=dtype).to(device=device) camera_loss.trans_estimation[:] = init_t loss = fitting.create_loss(loss_type=loss_type, joint_weights=joint_weights, rho=rho, use_joints_conf=use_joints_conf, use_face=use_face, use_hands=use_hands, vposer=vposer, pose_embedding=pose_embedding, body_pose_prior=body_pose_prior, shape_prior=shape_prior, angle_prior=angle_prior, expr_prior=expr_prior, left_hand_prior=left_hand_prior, right_hand_prior=right_hand_prior, jaw_prior=jaw_prior, interpenetration=interpenetration, pen_distance=pen_distance, search_tree=search_tree, tri_filtering_module=filter_faces, dtype=dtype, **kwargs) loss = loss.to(device=device) print("tipe img") print(type(img)) with fitting.FittingMonitor( batch_size=batch_size, visualize=visualize, background_image = img, camera = camera, **kwargs) as monitor: img = torch.tensor(img, dtype=dtype) H, W, _ = img.shape data_weight = 1000 / H # The closure passed to the optimizer camera_loss.reset_loss_weights({'data_weight': data_weight}) # Reset the parameters to estimate the initial translation of the # body model body_model.reset_params(body_pose=body_mean_pose) # If the distance between the 2D shoulders is smaller than a # predefined threshold then try 2 fits, the initial one and a 180 # degree rotation shoulder_dist = torch.dist(gt_joints[:, left_shoulder_idx], gt_joints[:, right_shoulder_idx]) try_both_orient = shoulder_dist.item() < side_view_thsh # Update the value of the translation of the camera as well as # the image center. with torch.no_grad(): camera.translation[:] = init_t.view_as(camera.translation) camera.center[:] = torch.tensor([W, H], dtype=dtype) * 0.5 # Re-enable gradient calculation for the camera translation camera.translation.requires_grad = True camera_opt_params = [camera.translation, body_model.global_orient] camera_optimizer, camera_create_graph = optim_factory.create_optimizer( camera_opt_params, **kwargs) # The closure passed to the optimizer fit_camera = monitor.create_fitting_closure( camera_optimizer, body_model, camera, gt_joints, camera_loss, create_graph=camera_create_graph, use_vposer=use_vposer, vposer=vposer, pose_embedding=pose_embedding, return_full_pose=False, return_verts=False) # Step 1: Optimize over the torso joints the camera translation # Initialize the computational graph by feeding the initial translation # of the camera and the initial pose of the body model. camera_init_start = time.time() cam_init_loss_val = monitor.run_fitting(camera_optimizer, fit_camera, camera_opt_params, body_model, use_vposer=use_vposer, pose_embedding=pose_embedding, vposer=vposer) if interactive: if use_cuda and torch.cuda.is_available(): torch.cuda.synchronize() tqdm.write('Camera initialization done after {:.4f}'.format( time.time() - camera_init_start)) tqdm.write('Camera initialization final loss {:.4f}'.format( cam_init_loss_val)) # If the 2D detections/positions of the shoulder joints are too # close the rotate the body by 180 degrees and also fit to that # orientation if try_both_orient: body_orient = body_model.global_orient.detach().cpu().numpy() flipped_orient = cv2.Rodrigues(body_orient)[0].dot( cv2.Rodrigues(np.array([0., np.pi, 0]))[0]) flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel() flipped_orient = torch.tensor(flipped_orient, dtype=dtype, device=device).unsqueeze(dim=0) orientations = [body_orient, flipped_orient] else: orientations = [body_model.global_orient.detach().cpu().numpy()] # store here the final error for both orientations, # and pick the orientation resulting in the lowest error results = [] # Step 2: Optimize the full model final_loss_val = 0 for or_idx, orient in enumerate(tqdm(orientations, desc='Orientation')): opt_start = time.time() new_params = defaultdict(global_orient=orient, body_pose=body_mean_pose) body_model.reset_params(**new_params) if use_vposer: with torch.no_grad(): pose_embedding.fill_(0) for opt_idx, curr_weights in enumerate(tqdm(opt_weights, desc='Stage')): body_params = list(body_model.parameters()) final_params = list( filter(lambda x: x.requires_grad, body_params)) if use_vposer: final_params.append(pose_embedding) body_optimizer, body_create_graph = optim_factory.create_optimizer( final_params, **kwargs) body_optimizer.zero_grad() curr_weights['data_weight'] = data_weight curr_weights['bending_prior_weight'] = ( 3.17 * curr_weights['body_pose_weight']) if use_hands: joint_weights[:, 25:76] = curr_weights['hand_weight'] if use_face: joint_weights[:, 76:] = curr_weights['face_weight'] loss.reset_loss_weights(curr_weights) closure = monitor.create_fitting_closure( body_optimizer, body_model, camera=camera, gt_joints=gt_joints, joints_conf=joints_conf, joint_weights=joint_weights, loss=loss, create_graph=body_create_graph, use_vposer=use_vposer, vposer=vposer, pose_embedding=pose_embedding, return_verts=True, return_full_pose=True) if interactive: if use_cuda and torch.cuda.is_available(): torch.cuda.synchronize() stage_start = time.time() final_loss_val = monitor.run_fitting( body_optimizer, closure, final_params, body_model, pose_embedding=pose_embedding, vposer=vposer, use_vposer=use_vposer) if interactive: if use_cuda and torch.cuda.is_available(): torch.cuda.synchronize() elapsed = time.time() - stage_start if interactive: tqdm.write('Stage {:03d} done after {:.4f} seconds'.format( opt_idx, elapsed)) if interactive: if use_cuda and torch.cuda.is_available(): torch.cuda.synchronize() elapsed = time.time() - opt_start tqdm.write( 'Body fitting Orientation {} done after {:.4f} seconds'.format( or_idx, elapsed)) tqdm.write('Body final loss val = {:.5f}'.format( final_loss_val)) # Get the result of the fitting process # Store in it the errors list in order to compare multiple # orientations, if they exist result = {'camera_' + str(key): val.detach().cpu().numpy() for key, val in camera.named_parameters()} result.update({key: val.detach().cpu().numpy() for key, val in body_model.named_parameters()}) if use_vposer: result['body_pose'] = pose_embedding.detach().cpu().numpy() results.append({'loss': final_loss_val, 'result': result}) with open(result_fn, 'wb') as result_file: if len(results) > 1: min_idx = (0 if results[0]['loss'] < results[1]['loss'] else 1) else: min_idx = 0 pickle.dump(results[min_idx]['result'], result_file, protocol=2) if save_meshes or visualize: body_pose = vposer.decode( pose_embedding, output_type='aa').view(1, -1) if use_vposer else None model_type = kwargs.get('model_type', 'smpl') append_wrists = model_type == 'smpl' and use_vposer if append_wrists: wrist_pose = torch.zeros([body_pose.shape[0], 6], dtype=body_pose.dtype, device=body_pose.device) body_pose = torch.cat([body_pose, wrist_pose], dim=1) model_output = body_model(return_verts=True, body_pose=body_pose) vertices = model_output.vertices.detach().cpu().numpy().squeeze() import trimesh out_mesh = trimesh.Trimesh(vertices, body_model.faces) rot = trimesh.transformations.rotation_matrix( np.radians(180), [1, 0, 0]) out_mesh.apply_transform(rot) out_mesh.export(mesh_fn) visualize=True if visualize: import pyrender material = pyrender.MetallicRoughnessMaterial( metallicFactor=0.0, alphaMode='OPAQUE', baseColorFactor=(1.0, 1.0, 0.9, 1.0)) mesh = pyrender.Mesh.from_trimesh( out_mesh, material=material) scene = pyrender.Scene(bg_color=[0.0, 0.0, 0.0, 0.0], ambient_light=(0.3, 0.3, 0.3)) scene.add(mesh, 'mesh') camera_center = camera.center.detach().cpu().numpy().squeeze() camera_transl = camera.translation.detach().cpu().numpy().squeeze() # Equivalent to 180 degrees around the y-axis. Transforms the fit to # OpenGL compatible coordinate system. camera_transl[0] *= -1.0 camera_pose = np.eye(4) camera_pose[:3, 3] = camera_transl camera = pyrender.camera.IntrinsicsCamera( fx=focal_length, fy=focal_length, cx=camera_center[0], cy=camera_center[1]) scene.add(camera, pose=camera_pose) # # Get the lights from the viewer light_nodes = create_raymond_lights() for node in light_nodes: scene.add_node(node) r = pyrender.OffscreenRenderer(viewport_width=W, viewport_height=H, point_size=1.0) color, _ = r.render(scene, flags=pyrender.RenderFlags.RGBA) color = color.astype(np.float32) / 255.0 valid_mask = (color[:, :, -1] > 0)[:, :, np.newaxis] input_img = img.detach().cpu().numpy() output_img = (color[:, :, :-1] * valid_mask + (1 - valid_mask) * input_img) img = pil_img.fromarray((output_img * 255).astype(np.uint8)) print(out_img_fn) print(img) img.save(out_img_fn)
def test_simple_input_with_scalar_baseline_conductance(self) -> None: net = BasicModel_MultiLayer() inp = torch.tensor([[0.0, 100.0, 0.0]]) self._conductance_test_assert( net, net.linear0, inp, [[0.0, 390.0, 0.0]], baselines=0.0 )
def create_dataset(args, processor, retrievers, relation_list, evaluate, input_dir): if args.local_rank not in [-1, 0]: # Make sure only the first process in distributed training process the dataset, and the others will use the cache torch.distributed.barrier() definition_info = DefinitionInfo() tokenizer, _ = configure_tokenizer_model(args, logger, retrievers, is_preprocess=True) logger.info("tokenizer: {}".format(tokenizer)) if args.test: temp_mark = "test" elif evaluate: temp_mark = "dev" else: temp_mark = "train" cached_features_file = os.path.join( input_dir, "cached_{}_{}_{}".format( temp_mark, args.model_type, str(args.cache_file_suffix), ), ) if os.path.exists(cached_features_file): logger.warning("cache file exist and exit program") exit() logger.info("Creating features from dataset file at %s", input_dir) if not os.path.exists(cached_features_file + "_example"): if args.test: examples = processor.get_test_examples(args.data_dir, filename=args.predict_file) else: examples = processor.get_dev_examples(args.data_dir, filename=args.predict_file) torch.save(examples, cached_features_file + "_example") else: logger.info("Loading examples from cached files.") examples = torch.load(cached_features_file + "_example") examples_tokenized = processor.tokenization_on_examples(examples, tokenizer, is_testing=args.test) features = processor.convert_examples_to_features(args, examples_tokenized, tokenizer, retrievers, not evaluate, debug=args.debug) features, dataset, all_kgs_graphs = processor.pad_and_index_features_all( features, retrievers, args, tokenizer, relation_list, encoder=None, definition_info=definition_info, is_training=not evaluate, debug=args.debug) if args.local_rank in [-1, 0]: for num in range(2): all_kgs_graphs_label_dict = {"glabel": torch.tensor([i for i in range(len(all_kgs_graphs))])} tmp_all_kgs_graphs = [gg[num] for gg in all_kgs_graphs] save_graphs(cached_features_file + "_all_kgs_graphs_choice_{}.bin".format(num), tmp_all_kgs_graphs, all_kgs_graphs_label_dict) logger.info("complete data preprocessing") logger.info("Saving features into cached file %s", cached_features_file) for f in features: del f.kgs_conceptids2synset_list torch.save({"features": features, "dataset": dataset, "examples": examples_tokenized}, cached_features_file) logger.info("Saving knowledge graph retrievers") for kg, retriever in retrievers.items(): if not os.path.exists(os.path.join(input_dir, args.kg_paths[kg])): os.mkdir(os.path.join(input_dir, args.kg_paths[kg])) torch.save(retriever, os.path.join(input_dir, args.kg_paths[kg], kg + args.cache_file_suffix)) logger.info("data create is done") if args.local_rank == 0: # Make sure only the first process in distributed training process the dataset, and the others will use the cache torch.distributed.barrier()
def __init__( self, n_input: int, n_batch: int = 0, n_labels: int = 0, n_hidden: int = 128, n_latent: int = 10, n_layers: int = 1, dropout_rate: float = 0.1, dispersion: str = "gene", log_variational: bool = True, gene_likelihood: str = "zinb", y_prior=None, labels_groups: Sequence[int] = None, use_labels_groups: bool = False, classifier_parameters: dict = dict(), ): super().__init__( n_input, n_hidden=n_hidden, n_latent=n_latent, n_layers=n_layers, dropout_rate=dropout_rate, n_batch=n_batch, dispersion=dispersion, log_variational=log_variational, gene_likelihood=gene_likelihood, ) self.n_labels = n_labels # Classifier takes n_latent as input cls_parameters = { "n_layers": n_layers, "n_hidden": n_hidden, "dropout_rate": dropout_rate, } cls_parameters.update(classifier_parameters) self.classifier = Classifier(n_latent, n_labels=n_labels, **cls_parameters) self.encoder_z2_z1 = Encoder( n_latent, n_latent, n_cat_list=[self.n_labels], n_layers=n_layers, n_hidden=n_hidden, dropout_rate=dropout_rate, ) self.decoder_z1_z2 = Decoder( n_latent, n_latent, n_cat_list=[self.n_labels], n_layers=n_layers, n_hidden=n_hidden, ) self.y_prior = torch.nn.Parameter( y_prior if y_prior is not None else (1 / n_labels) * torch.ones(1, n_labels), requires_grad=False, ) self.use_labels_groups = use_labels_groups self.labels_groups = (np.array(labels_groups) if labels_groups is not None else None) if self.use_labels_groups: assert labels_groups is not None, "Specify label groups" unique_groups = np.unique(self.labels_groups) self.n_groups = len(unique_groups) assert (unique_groups == np.arange(self.n_groups)).all() self.classifier_groups = Classifier(n_latent, n_hidden, self.n_groups, n_layers, dropout_rate) self.groups_index = torch.nn.ParameterList([ torch.nn.Parameter( torch.tensor( (self.labels_groups == i).astype(np.uint8), dtype=torch.uint8, ), requires_grad=False, ) for i in range(self.n_groups) ])
def train_one_epoch( cls, model: torch.nn.Module, iterator: Iterable[Tuple[List[str], Dict[str, torch.Tensor]]], optimizers: Sequence[torch.optim.Optimizer], schedulers: Sequence[Optional[AbsScheduler]], reporter: SubReporter, options: TrainerOptions, ) -> bool: assert check_argument_types() # Note(kamo): assumes one optimizer assert cls.num_optimizers == 1, cls.num_optimizers assert len(optimizers) == 1, len(optimizers) optimizer = optimizers[0] scheduler = schedulers[0] grad_noise = options.grad_noise accum_grad = options.accum_grad grad_clip = options.grad_clip log_interval = options.log_interval no_forward_run = options.no_forward_run ngpu = options.ngpu distributed = isinstance(model, torch.nn.parallel.DistributedDataParallel) use_apex = options.train_dtype in ("O0", "O1", "O2", "O3") if log_interval is None: try: log_interval = max(len(iterator) // 20, 10) except TypeError: log_interval = 100 model.train() all_steps_are_invalid = True # [For distributed] Because iteration counts are not always equals between # processes, send stop-flag to the other processes if iterator is finished iterator_stop = torch.tensor(0).to("cuda" if ngpu > 0 else "cpu") start_time = time.perf_counter() for iiter, (_, batch) in enumerate( reporter.measure_iter_time(iterator, "iter_time"), 1): assert isinstance(batch, dict), type(batch) if distributed: torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) if iterator_stop > 0: break batch = to_device(batch, "cuda" if ngpu > 0 else "cpu") if no_forward_run: all_steps_are_invalid = False reporter.register({}) continue with reporter.measure_time("forward_time"): loss, stats, weight = model(**batch) if ngpu > 1 or distributed: # Apply weighted averaging for loss and stats loss = (loss * weight.type(loss.dtype)).sum() # if distributed, this method can also apply all_reduce() stats, weight = recursive_average(stats, weight, distributed) # Now weight is summation over all workers loss /= weight if distributed: # NOTE(kamo): Multiply world_size because DistributedDataParallel # automatically normalizes the gradient by world_size. loss *= torch.distributed.get_world_size() reporter.register(stats, weight) loss /= accum_grad with reporter.measure_time("backward_time"): if use_apex: try: from apex import amp except ImportError: logging.error( "You need to install apex. " "See https://github.com/NVIDIA/apex#linux") with amp.scale_loss(loss, optimizers) as scaled_loss: scaled_loss.backward() else: loss.backward() if iiter % accum_grad == 0: # gradient noise injection if grad_noise: add_gradient_noise( model, reporter.get_total_count(), duration=100, eta=1.0, scale_factor=0.55, ) # compute the gradient norm to check if it is normal or not grad_norm = torch.nn.utils.clip_grad_norm_( model.parameters(), grad_clip) # PyTorch<=1.4, clip_grad_norm_ returns float value if not isinstance(grad_norm, torch.Tensor): grad_norm = torch.tensor(grad_norm) if not torch.isfinite(grad_norm): logging.warning( f"The grad norm is {grad_norm}. Skipping updating the model." ) else: all_steps_are_invalid = False with reporter.measure_time("optim_step_time"): optimizer.step() if isinstance(scheduler, AbsBatchStepScheduler): scheduler.step() optimizer.zero_grad() # Register lr and train/load time[sec/step], # where step refers to accum_grad * mini-batch reporter.register( dict( { f"lr_{i}": pg["lr"] for i, pg in enumerate(optimizer.param_groups) if "lr" in pg }, train_time=time.perf_counter() - start_time, ), # Suppress to increment the internal counter. not_increment_count=True, ) start_time = time.perf_counter() if iiter % log_interval == 0: logging.info(reporter.log_message()) else: if distributed: iterator_stop.fill_(1) torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) return all_steps_are_invalid
def preprocess(self, state): state = torch.tensor(np.expand_dims(state, 0)).to(self.device) return state.float() / 256
print("done") print(len(index2word)) print(len(word2index)) unkvec = word2vec["the"].copy() padvec = word2vec["apple"].copy() random.shuffle(unkvec) random.shuffle(padvec) avec = word2vec["a"] at = torch.tensor([avec]) print(at.size()) l = [] for index in range(0, len(word2vec)): if index % 100000 == 0: print(index) vector = word2vec[index2word[index]] l.append(vector) l.append(unkvec) l.append(padvec) embmatrix = torch.tensor(l)
def _images_to_observation(images, bit_depth): images = torch.tensor(cv2.resize(images, (64, 64), interpolation=cv2.INTER_LINEAR).transpose(2, 0, 1), dtype=torch.float32) # Resize and put channel first preprocess_observation_(images, bit_depth) # Quantise, centre and dequantise inplace return images.unsqueeze(dim=0) # Add batch dimension
out = z.mean() print(z, out) a = torch.randn(2, 2) a = ((a * 3) / (a - 1)) print(a.requires_grad) a.requires_grad_(True) print(a.requires_grad) b = (a * a).sum() print(b.grad_fn) print(x.grad) out.backward() print(x.grad) x = torch.randn(3, requires_grad=True) y = x * 2 while y.data.norm() < 1000: y = y * 2 print(y) gradients = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float) y.backward(gradients) print(x.grad) print(x.requires_grad) print((x ** 2).requires_grad) with torch.no_grad(): print((x ** 2).requires_grad)
def __init__(self, loss, weight=1.0): super().__init__() self.loss = loss self.register_buffer("weight", torch.tensor([weight]))
def train_student( self, epochs=10, plot_losses=True, save_model=True, save_model_pth="./models/student.pth", ): """ Function that will be training the student :param epochs (int): Number of epochs you want to train the teacher :param plot_losses (bool): True if you want to plot the losses :param save_model (bool): True if you want to save the student model :param save_model_pth (str): Path where you want to save the student model """ self.teacher_distill_loader = self._get_teacher_dataloaders( batch_size=self.train_loader.batch_size, mode="distill") y_pred_teacher = [] print("Obtaining teacher predictions...") self.teacher_model.eval() self.teacher_model.to(self.device) for batch in self.teacher_distill_loader: b_input_ids = batch[0].to(self.device) b_input_mask = batch[1].to(self.device) b_labels = batch[2].to(self.device) with torch.no_grad(): (loss, logits) = self.teacher_model( b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels, ) logits = logits.detach().cpu().numpy() y_pred_teacher.append(logits) self.student_model.train() loss_arr = [] length_of_dataset = len(self.train_loader.dataset) best_acc = 0.0 self.best_student_model_weights = deepcopy( self.student_model.state_dict()) self.student_model.to(self.device) print("\nTraining student...") for ep in range(epochs): print("") print("======== Epoch {:} / {:} ========".format(ep + 1, epochs)) epoch_loss = 0.0 correct = 0 for (data, data_len, label), bert_prob in zip(self.train_loader, y_pred_teacher): data = data.to(self.device) data_len = data_len.to(self.device) label = label.to(self.device) bert_prob = torch.tensor(bert_prob, dtype=torch.float) teacher_out = bert_prob.to(self.device) self.optimizer_student.zero_grad() student_out = self.student_model(data, data_len).squeeze(1) loss = self.calculate_kd_loss(student_out, teacher_out, label) pred = student_out.argmax(dim=1, keepdim=True) correct += pred.eq(label.view_as(pred)).sum().item() loss.backward() # ##For preventing exploding gradients # torch.nn.utils.clip_grad_norm_(self.student_model.parameters(), 1.0) self.optimizer_student.step() epoch_loss += loss epoch_acc = correct / length_of_dataset print(f"Loss: {epoch_loss} | Accuracy: {epoch_acc}") epoch_acc = correct / length_of_dataset if epoch_acc > best_acc: best_acc = epoch_acc self.best_student_model_weights = deepcopy( self.student_model.state_dict()) if self.log: self.writer.add_scalar("Training loss/Student", epoch_loss, epochs) self.writer.add_scalar("Training accuracy/Student", epoch_acc, epochs) loss_arr.append(epoch_loss) print(f"Epoch: {ep+1}, Loss: {epoch_loss}, Accuracy: {epoch_acc}") self.student_model.load_state_dict(self.best_student_model_weights) if save_model: torch.save(self.student_model.state_dict(), save_model_pth) if plot_losses: plt.plot(loss_arr)
def test_simple_input_conductance(self): net = BasicModel_MultiLayer() inp = torch.tensor([[0.0, 100.0, 0.0]]) self._conductance_test_assert(net, net.linear0, inp, [[0.0, 390.0, 0.0]])
def step(self, actions): observations, rewards, dones = zip(*[env.step(action) for env, action in zip(self.envs, actions)]) self.dones = dones return torch.cat(observations), torch.tensor(rewards, dtype=torch.float32), torch.tensor(dones, dtype=torch.uint8)