def test_forward(self): batch = 16 len1, len2 = 21, 24 seq_len1 = torch.randint(low=len1 - 10, high=len1 + 1, size=(batch,)).long() seq_len2 = torch.randint(low=len2 - 10, high=len2 + 1, size=(batch,)).long() mask1 = [] for w in seq_len1: mask1.append([1] * w.item() + [0] * (len1 - w.item())) mask1 = torch.FloatTensor(mask1) mask2 = [] for w in seq_len2: mask2.append([1] * w.item() + [0] * (len2 - w.item())) mask2 = torch.FloatTensor(mask2) d = 200 # hidden dimension l = 20 # number of perspective test1 = torch.randn(batch, len1, d) test2 = torch.randn(batch, len2, d) test1 = test1 * mask1.view(-1, len1, 1).expand(-1, len1, d) test2 = test2 * mask2.view(-1, len2, 1).expand(-1, len2, d) test1_fw, test1_bw = torch.split(test1, d // 2, dim=-1) test2_fw, test2_bw = torch.split(test2, d // 2, dim=-1) ml_fw = BiMpmMatching.from_params(Params({"is_forward": True, "num_perspectives": l})) ml_bw = BiMpmMatching.from_params(Params({"is_forward": False, "num_perspectives": l})) vecs_p_fw, vecs_h_fw = ml_fw(test1_fw, mask1, test2_fw, mask2) vecs_p_bw, vecs_h_bw = ml_bw(test1_bw, mask1, test2_bw, mask2) vecs_p, vecs_h = torch.cat(vecs_p_fw + vecs_p_bw, dim=2), torch.cat(vecs_h_fw + vecs_h_bw, dim=2) assert vecs_p.size() == torch.Size([batch, len1, 10 + 10 * l]) assert vecs_h.size() == torch.Size([batch, len2, 10 + 10 * l]) assert ml_fw.get_output_dim() == ml_bw.get_output_dim() == vecs_p.size(2) // 2 == vecs_h.size(2) // 2
def dummy_inputs(cls, params, init_case): n_unique_src_words = 13 scores = torch.randn((params["batch_size"] * params["tgt_max_len"], init_case["vocab_size"] + n_unique_src_words)) scores = softmax(scores, dim=1) align = torch.randint(0, n_unique_src_words, (params["batch_size"] * params["tgt_max_len"],)) target = torch.randint(0, init_case["vocab_size"], (params["batch_size"] * params["tgt_max_len"],)) target[0] = init_case["unk_index"] target[1] = init_case["ignore_index"] return scores, align, target
def create_fake_data(shapes, vectorizers, order, min_=0, max_=50,): data = { k: torch.randint(min_, max_, shapes[type(v)]) for k, v in vectorizers.items() } ordered_data = tuple(data[k] for k in order) lengths = torch.LongTensor([data[list(data.keys())[0]].shape[1]]) return ordered_data, lengths
def test_repeating_excluded_index_does_not_die(self): # batch 0 will repeat excluded idx, batch 1 will repeat n_words = 100 repeat_idx = 47 # will be repeated and should be blocked repeat_idx_ignored = 7 # will be repeated and should not be blocked ngram_repeat = 3 for batch_sz in [1, 3, 17]: samp = RandomSampling( 0, 1, 2, batch_sz, torch.device("cpu"), 0, ngram_repeat, {repeat_idx_ignored}, False, 30, 1., 5, torch.randint(0, 30, (batch_sz,))) for i in range(ngram_repeat + 4): word_probs = torch.full( (batch_sz, n_words), -float('inf')) word_probs[0, repeat_idx_ignored] = 0 if batch_sz > 1: word_probs[1, repeat_idx] = 0 word_probs[2:, repeat_idx + i] = 0 attns = torch.randn(1, batch_sz, 53) samp.advance(word_probs, attns) if i <= ngram_repeat: self.assertFalse(samp.topk_scores.eq( self.BLOCKED_SCORE).any()) else: # now batch 1 dies self.assertFalse(samp.topk_scores[0].eq( self.BLOCKED_SCORE).any()) if batch_sz > 1: self.assertTrue(samp.topk_scores[1].eq( self.BLOCKED_SCORE).all()) self.assertFalse(samp.topk_scores[2:].eq( self.BLOCKED_SCORE).any())
def test_advance_with_all_repeats_gets_blocked(self): # all beams repeat (beam >= 1 repeat dummy scores) beam_sz = 5 n_words = 100 repeat_idx = 47 ngram_repeat = 3 for batch_sz in [1, 3]: beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), 0, 30, False, ngram_repeat, set(), torch.randint(0, 30, (batch_sz,)), False, 0.) for i in range(ngram_repeat + 4): # predict repeat_idx over and over again word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) word_probs[0::beam_sz, repeat_idx] = 0 attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i <= ngram_repeat: expected_scores = torch.tensor( [0] + [-float('inf')] * (beam_sz - 1))\ .repeat(batch_sz, 1) self.assertTrue(beam.topk_log_probs.equal(expected_scores)) else: self.assertTrue( beam.topk_log_probs.equal( torch.tensor(self.BLOCKED_SCORE) .repeat(batch_sz, beam_sz)))
def PyTorchOperatorTestCase(test_name, op_type, input_shapes, op_args, run_mode): """Benchmark Tester function for Pytorch framework. """ inputs = [] is_contig = 'contig' not in op_args or op_args['contig'] dtype = op_args['dtype'] if 'dtype' in op_args else torch.float32 for shape in input_shapes: tensor_shape = list(shape) if not is_contig: tensor_shape = [s * 2 for s in tensor_shape] if dtype in [torch.float32, torch.float64]: # skip float16 input = torch.rand(tensor_shape, dtype=dtype) elif not dtype.is_floating_point: input = torch.randint(low=0, high=100, size=tensor_shape, dtype=dtype) else: input = torch.ones(tensor_shape, dtype=dtype) if not is_contig: slices = [] for dim in tensor_shape: slices.append(slice(0, dim, 2)) input = input[slices] assert list(input.size()) == list(shape) assert not input.is_contiguous() inputs.append(input) def benchmark_func(num_runs): op_type(*(inputs + [num_runs])) benchmark_core.add_benchmark_tester("PyTorch", test_name, input_shapes, op_args, run_mode, benchmark_func)
def pytorch_net_to_buffer(pytorch_net, input_dim, model_on_gpu, float_input=True): """Traces a pytorch net and outputs a python buffer object holding net.""" training = pytorch_net.training pytorch_net.train(False) for name, p in pytorch_net.named_parameters(): inf_count = torch.isinf(p).sum().item() nan_count = torch.isnan(p).sum().item() assert inf_count + nan_count == 0, "{} has {} inf and {} nan".format( name, inf_count, nan_count ) if float_input: dtype = torch.cuda.FloatTensor if model_on_gpu else torch.FloatTensor dummy_input = torch.randn(1, input_dim).type(dtype) else: dtype = torch.cuda.LongTensor if model_on_gpu else torch.LongTensor dummy_input = torch.randint(low=0, high=1, size=(1, input_dim)).type(dtype) write_buffer = BytesIO() try: torch.onnx.export(pytorch_net, dummy_input, write_buffer) finally: pytorch_net.train(training) return write_buffer
def test_rescale_torch_tensor(self): rows, cols = 3, 5 original_tensor = torch.randint(low=10, high=40, size=(rows, cols)).float() prev_max_tensor = torch.ones(1, 5) * 40.0 prev_min_tensor = torch.ones(1, 5) * 10.0 new_min_tensor = torch.ones(1, 5) * -1.0 new_max_tensor = torch.ones(1, 5).float() print("Original tensor: ", original_tensor) rescaled_tensor = rescale_torch_tensor( original_tensor, new_min_tensor, new_max_tensor, prev_min_tensor, prev_max_tensor, ) print("Rescaled tensor: ", rescaled_tensor) reconstructed_original_tensor = rescale_torch_tensor( rescaled_tensor, prev_min_tensor, prev_max_tensor, new_min_tensor, new_max_tensor, ) print("Reconstructed Original tensor: ", reconstructed_original_tensor) comparison_tensor = torch.eq(original_tensor, reconstructed_original_tensor) self.assertTrue(torch.sum(comparison_tensor), rows * cols)
def predict_fn(input_data, model): logger.info('Generating text based on input parameters.') corpus = model['corpus'] model = model['model'] device = torch.device("cuda" if torch.cuda.is_available() else "cpu") logger.info('Current device: {}'.format(device)) torch.manual_seed(input_data['seed']) ntokens = len(corpus.dictionary) input = torch.randint(ntokens, (1, 1), dtype=torch.long).to(device) hidden = model.init_hidden(1) logger.info('Generating {} words.'.format(input_data['words'])) result = [] with torch.no_grad(): # no tracking history for i in range(input_data['words']): output, hidden = model(input, hidden) word_weights = output.squeeze().div(input_data['temperature']).exp().cpu() word_idx = torch.multinomial(word_weights, 1)[0] input.fill_(word_idx) word = corpus.dictionary.idx2word[word_idx] word = word if type(word) == str else word.decode() if word == '<eos>': word = '\n' elif i % 12 == 11: word = word + '\n' else: word = word + ' ' result.append(word) return ''.join(result)
def pad_inputs(cls, params): lengths = torch.randint(1, params["max_len"], (params["batch_size"],)).tolist() lengths[params["full_length_seq"]] = params["max_len"] fake_input = [ torch.randn((params["nfeats"], lengths[b])) for b in range(params["batch_size"])] return fake_input, lengths
def test_sampled_softmax_can_run(self): softmax = SampledSoftmaxLoss(num_words=1000, embedding_dim=12, num_samples=50) # sequence_length, embedding_dim embedding = torch.rand(100, 12) targets = torch.randint(0, 1000, (100,)).long() _ = softmax(embedding, targets)
def dummy_inputs(cls, params, init_case): max_seq_len = params["max_seq_len"] batch_size = params["batch_size"] fv_sizes = init_case["feat_vocab_sizes"] n_words = init_case["word_vocab_size"] voc_sizes = [n_words] + fv_sizes pad_idxs = [init_case["word_padding_idx"]] + \ init_case["feat_padding_idx"] lengths = torch.randint(0, max_seq_len, (batch_size,)) lengths[0] = max_seq_len inps = torch.empty((max_seq_len, batch_size, len(voc_sizes)), dtype=torch.long) for f, (voc_size, pad_idx) in enumerate(zip(voc_sizes, pad_idxs)): for b, len_ in enumerate(lengths): inps[:len_, b, f] = torch.randint(0, voc_size-1, (len_,)) inps[len_:, b, f] = pad_idx return inps
def test_beam_is_done_when_n_best_beams_eos_using_min_length(self): # this is also a test that when block_ngram_repeat=0, # repeating is acceptable beam_sz = 5 batch_sz = 3 n_words = 100 _non_eos_idxs = [47, 51, 13, 88, 99] valid_score_dist = torch.log_softmax(torch.tensor( [6., 5., 4., 3., 2., 1.]), dim=0) min_length = 5 eos_idx = 2 beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), min_length, 30, False, 0, set(), torch.randint(0, 30, (batch_sz,)), False, 0.) for i in range(min_length + 4): # non-interesting beams are going to get dummy values word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) if i == 0: # "best" prediction is eos - that should be blocked word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] # include at least beam_sz predictions OTHER than EOS # that are greater than -1e20 for j, score in zip(_non_eos_idxs, valid_score_dist[1:]): word_probs[0::beam_sz, j] = score elif i <= min_length: # predict eos in beam 1 word_probs[1::beam_sz, eos_idx] = valid_score_dist[0] # provide beam_sz other good predictions in other beams for k, (j, score) in enumerate( zip(_non_eos_idxs, valid_score_dist[1:])): beam_idx = min(beam_sz-1, k) word_probs[beam_idx::beam_sz, j] = score else: word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] word_probs[1::beam_sz, eos_idx] = valid_score_dist[0] # provide beam_sz other good predictions in other beams for k, (j, score) in enumerate( zip(_non_eos_idxs, valid_score_dist[1:])): beam_idx = min(beam_sz-1, k) word_probs[beam_idx::beam_sz, j] = score attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i < min_length: self.assertFalse(beam.done) elif i == min_length: # beam 1 dies on min_length self.assertTrue(beam.is_finished[:, 1].all()) beam.update_finished() self.assertFalse(beam.done) else: # i > min_length # beam 0 dies on the step after beam 1 dies self.assertTrue(beam.is_finished[:, 0].all()) beam.update_finished() self.assertTrue(beam.done)
def sample_code(num, cat_dim=0, cont_dim=0, bin_dim=0, device=None) -> torch.Tensor: cat_onehot = cont = bin = None if cat_dim > 0: cat = torch.randint(cat_dim, size=(num, 1), dtype=torch.long, device=device) cat_onehot = torch.zeros(num, cat_dim, dtype=torch.float, device=device) cat_onehot.scatter_(1, cat, 1) if cont_dim > 0: cont = 2. * torch.rand(num, cont_dim, device=device) - 1. if bin_dim > 0: bin = (torch.rand(num, bin_dim, device=device) > .5).float() return torch.cat([x for x in [cat_onehot, cont, bin] if x is not None], 1)
def test_beam_advance_against_known_reference(self): beam = BeamSearch( self.BEAM_SZ, self.BATCH_SZ, 0, 1, 2, self.N_BEST, torch.device("cpu"), GlobalScorerStub(), 0, 30, False, 0, set(), torch.randint(0, 30, (self.BATCH_SZ,)), False, 0.) expected_beam_scores = self.init_step(beam, 1) expected_beam_scores = self.first_step(beam, expected_beam_scores, 1) expected_beam_scores = self.second_step(beam, expected_beam_scores, 1) self.third_step(beam, expected_beam_scores, 1)
def test_doesnt_predict_eos_if_shorter_than_min_len(self): # beam 0 will always predict EOS. The other beams will predict # non-eos scores. for batch_sz in [1, 3]: beam_sz = 5 n_words = 100 _non_eos_idxs = [47, 51, 13, 88, 99] valid_score_dist = torch.log_softmax(torch.tensor( [6., 5., 4., 3., 2., 1.]), dim=0) min_length = 5 eos_idx = 2 lengths = torch.randint(0, 30, (batch_sz,)) beam = BeamSearch(beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), min_length, 30, False, 0, set(), lengths, False, 0.) all_attns = [] for i in range(min_length + 4): # non-interesting beams are going to get dummy values word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) if i == 0: # "best" prediction is eos - that should be blocked word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] # include at least beam_sz predictions OTHER than EOS # that are greater than -1e20 for j, score in zip(_non_eos_idxs, valid_score_dist[1:]): word_probs[0::beam_sz, j] = score else: # predict eos in beam 0 word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] # provide beam_sz other good predictions for k, (j, score) in enumerate( zip(_non_eos_idxs, valid_score_dist[1:])): beam_idx = min(beam_sz-1, k) word_probs[beam_idx::beam_sz, j] = score attns = torch.randn(1, batch_sz * beam_sz, 53) all_attns.append(attns) beam.advance(word_probs, attns) if i < min_length: expected_score_dist = \ (i+1) * valid_score_dist[1:].unsqueeze(0) self.assertTrue( beam.topk_log_probs.allclose( expected_score_dist)) elif i == min_length: # now the top beam has ended and no others have self.assertTrue(beam.is_finished[:, 0].eq(1).all()) self.assertTrue(beam.is_finished[:, 1:].eq(0).all()) else: # i > min_length # not of interest, but want to make sure it keeps running # since only beam 0 terminates and n_best = 2 pass
def test_beam_advance_against_known_reference(self): scorer = GNMTGlobalScorer(0.7, 0., "avg", "none") beam = BeamSearch( self.BEAM_SZ, self.BATCH_SZ, 0, 1, 2, self.N_BEST, torch.device("cpu"), scorer, 0, 30, False, 0, set(), torch.randint(0, 30, (self.BATCH_SZ,)), False, 0.) expected_beam_scores = self.init_step(beam, 1.) expected_beam_scores = self.first_step(beam, expected_beam_scores, 3) expected_beam_scores = self.second_step(beam, expected_beam_scores, 4) self.third_step(beam, expected_beam_scores, 5)
def setUpClass(cls): if not os.path.exists(cls._IMG_DATA_DIR): os.makedirs(cls._IMG_DATA_DIR) if not os.path.exists(cls._IMG_LIST_DIR): os.makedirs(cls._IMG_LIST_DIR) with open(cls._JUNK_FILE, "w") as f: f.write("this is some garbage\nShould have no impact.") with open(cls._IMG_LIST_PATHS_PATH, "w") as f_list_fnames, \ open(cls._IMG_LIST_FNAMES_PATH, "w") as f_list_paths: cls.n_rows = torch.randint(30, 314, (cls._N_EXAMPLES,)) cls.n_cols = torch.randint(30, 314, (cls._N_EXAMPLES,)) for i in range(cls._N_EXAMPLES): img = np.random.randint( 0, 255, (cls.n_rows[i], cls.n_cols[i], cls._N_CHANNELS)) f_path = cls._IMG_DATA_PATH_FMT.format(i) cv2.imwrite(f_path, img) f_name_short = cls._IMG_DATA_FMT.format(i) f_list_fnames.write(f_name_short + "\n") f_list_paths.write(f_path + "\n")
def test_repeating_excluded_index_does_not_die(self): # beam 0 and beam >= 2 will repeat (beam 2 repeats excluded idx) beam_sz = 5 n_words = 100 repeat_idx = 47 # will be repeated and should be blocked repeat_idx_ignored = 7 # will be repeated and should not be blocked ngram_repeat = 3 for batch_sz in [1, 3]: beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), 0, 30, False, ngram_repeat, {repeat_idx_ignored}, torch.randint(0, 30, (batch_sz,)), False, 0.) for i in range(ngram_repeat + 4): # non-interesting beams are going to get dummy values word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) if i == 0: word_probs[0::beam_sz, repeat_idx] = -0.1 word_probs[0::beam_sz, repeat_idx + i + 1] = -2.3 word_probs[0::beam_sz, repeat_idx_ignored] = -5.0 else: # predict the same thing in beam 0 word_probs[0::beam_sz, repeat_idx] = 0 # continue pushing around what beam 1 predicts word_probs[1::beam_sz, repeat_idx + i + 1] = 0 # predict the allowed-repeat again in beam 2 word_probs[2::beam_sz, repeat_idx_ignored] = 0 attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i <= ngram_repeat: self.assertFalse(beam.topk_log_probs[:, 0].eq( self.BLOCKED_SCORE).any()) self.assertFalse(beam.topk_log_probs[:, 1].eq( self.BLOCKED_SCORE).any()) self.assertFalse(beam.topk_log_probs[:, 2].eq( self.BLOCKED_SCORE).any()) else: # now beam 0 dies, beam 1 -> beam 0, beam 2 -> beam 1 # and the rest die self.assertFalse(beam.topk_log_probs[:, 0].eq( self.BLOCKED_SCORE).any()) # since all preds after i=0 are 0, we can check # that the beam is the correct idx by checking that # the curr score is the initial score self.assertTrue(beam.topk_log_probs[:, 0].eq(-2.3).all()) self.assertFalse(beam.topk_log_probs[:, 1].eq( self.BLOCKED_SCORE).all()) self.assertTrue(beam.topk_log_probs[:, 1].eq(-5.0).all()) self.assertTrue( beam.topk_log_probs[:, 2:].equal( torch.tensor(self.BLOCKED_SCORE) .repeat(batch_sz, beam_sz - 2)))
def numericalize_inputs(cls, init_case, params): bs = params["batch_size"] max_len = params["max_len"] lengths = torch.randint(1, max_len, (bs,)) lengths[params["full_length_seq"]] = max_len nfeats = params["nfeats"] fake_input = torch.full( (bs, 1, nfeats, max_len), init_case["pad_index"]) for b in range(bs): fake_input[b, :, :, :lengths[b]] = torch.randn( (1, nfeats, lengths[b])) if init_case["include_lengths"]: fake_input = (fake_input, lengths) return fake_input, lengths
def test_sampled_almost_equals_unsampled_when_num_samples_is_almost_all(self): sampled_softmax = SampledSoftmaxLoss(num_words=10000, embedding_dim=12, num_samples=9999) unsampled_softmax = _SoftmaxLoss(num_words=10000, embedding_dim=12) # sequence_length, embedding_dim embedding = torch.rand(100, 12) targets = torch.randint(0, 1000, (100,)).long() full_loss = unsampled_softmax(embedding, targets).item() sampled_loss = sampled_softmax(embedding, targets).item() # Should be really close pct_error = (sampled_loss - full_loss) / full_loss assert abs(pct_error) < 0.02
def test_advance_with_some_repeats_gets_blocked(self): # beam 0 and beam >=2 will repeat (beam >= 2 repeat dummy scores) beam_sz = 5 n_words = 100 repeat_idx = 47 ngram_repeat = 3 for batch_sz in [1, 3]: beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), 0, 30, False, ngram_repeat, set(), torch.randint(0, 30, (batch_sz,)), False, 0.) for i in range(ngram_repeat + 4): # non-interesting beams are going to get dummy values word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) if i == 0: # on initial round, only predicted scores for beam 0 # matter. Make two predictions. Top one will be repeated # in beam zero, second one will live on in beam 1. word_probs[0::beam_sz, repeat_idx] = -0.1 word_probs[0::beam_sz, repeat_idx + i + 1] = -2.3 else: # predict the same thing in beam 0 word_probs[0::beam_sz, repeat_idx] = 0 # continue pushing around what beam 1 predicts word_probs[1::beam_sz, repeat_idx + i + 1] = 0 attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i <= ngram_repeat: self.assertFalse( beam.topk_log_probs[0::beam_sz].eq( self.BLOCKED_SCORE).any()) self.assertFalse( beam.topk_log_probs[1::beam_sz].eq( self.BLOCKED_SCORE).any()) else: # now beam 0 dies (along with the others), beam 1 -> beam 0 self.assertFalse( beam.topk_log_probs[:, 0].eq( self.BLOCKED_SCORE).any()) self.assertTrue( beam.topk_log_probs[:, 1:].equal( torch.tensor(self.BLOCKED_SCORE) .repeat(batch_sz, beam_sz-1)))
def setUpClass(cls): if not os.path.exists(cls._AUDIO_DATA_DIR): os.makedirs(cls._AUDIO_DATA_DIR) if not os.path.exists(cls._AUDIO_LIST_DIR): os.makedirs(cls._AUDIO_LIST_DIR) with open(cls._JUNK_FILE, "w") as f: f.write("this is some garbage\nShould have no impact.") with open(cls._AUDIO_LIST_PATHS_PATH, "w") as f_list_fnames, \ open(cls._AUDIO_LIST_FNAMES_PATH, "w") as f_list_paths: lengths = torch.randint(int(.5e5), int(1.5e6), (cls._N_EXAMPLES,)) for i in range(cls._N_EXAMPLES): # dividing gets the noise in [-1, 1] white_noise = torch.randn((cls._N_CHANNELS, lengths[i])) / 10 f_path = cls._AUDIO_DATA_PATH_FMT.format(i) torchaudio.save(f_path, white_noise, cls._SAMPLE_RATE) f_name_short = cls._AUDIO_DATA_FMT.format(i) f_list_fnames.write(f_name_short + "\n") f_list_paths.write(f_path + "\n")
def varlen_lstm_inputs(minlen=30, maxlen=100, numLayers=1, inputSize=512, hiddenSize=512, miniBatch=64, return_module=False, device='cuda', seed=None, **kwargs): if seed is not None: torch.manual_seed(seed) lengths = torch.randint( low=minlen, high=maxlen, size=[miniBatch], dtype=torch.long, device=device) x = [torch.randn(length, inputSize, device=device) for length in lengths] hx = torch.randn(numLayers, miniBatch, hiddenSize, device=device) cx = torch.randn(numLayers, miniBatch, hiddenSize, device=device) lstm = torch.nn.LSTM(inputSize, hiddenSize, numLayers).to(device) if return_module: return x, lengths, (hx, cx), lstm.all_weights, lstm else: # NB: lstm.all_weights format: # wih, whh, bih, bhh = lstm.all_weights[layer] return x, lengths, (hx, cx), lstm.all_weights, None
def test_advance_with_some_repeats_gets_blocked(self): # batch 0 and 7 will repeat, the rest will advance n_words = 100 repeat_idx = 47 other_repeat_idx = 12 ngram_repeat = 3 for batch_sz in [1, 3, 13]: samp = RandomSampling( 0, 1, 2, batch_sz, torch.device("cpu"), 0, ngram_repeat, set(), False, 30, 1., 5, torch.randint(0, 30, (batch_sz,))) for i in range(ngram_repeat + 4): word_probs = torch.full( (batch_sz, n_words), -float('inf')) # predict the same thing in batch 0 and 7 every i word_probs[0, repeat_idx] = 0 if batch_sz > 7: word_probs[7, other_repeat_idx] = 0 # push around what the other batches predict word_probs[1:7, repeat_idx + i] = 0 if batch_sz > 7: word_probs[8:, repeat_idx + i] = 0 attns = torch.randn(1, batch_sz, 53) samp.advance(word_probs, attns) if i <= ngram_repeat: self.assertFalse( samp.topk_scores.eq( self.BLOCKED_SCORE).any()) else: # now batch 0 and 7 die self.assertTrue(samp.topk_scores[0].eq(self.BLOCKED_SCORE)) if batch_sz > 7: self.assertTrue(samp.topk_scores[7].eq( self.BLOCKED_SCORE)) self.assertFalse( samp.topk_scores[1:7].eq( self.BLOCKED_SCORE).any()) if batch_sz > 7: self.assertFalse( samp.topk_scores[8:].eq( self.BLOCKED_SCORE).any())
def test_doesnt_predict_eos_if_shorter_than_min_len(self): # batch 0 will always predict EOS. The other batches will predict # non-eos scores. for batch_sz in [1, 3]: n_words = 100 _non_eos_idxs = [47] valid_score_dist = torch.log_softmax(torch.tensor( [6., 5.]), dim=0) min_length = 5 eos_idx = 2 lengths = torch.randint(0, 30, (batch_sz,)) samp = RandomSampling( 0, 1, 2, batch_sz, torch.device("cpu"), min_length, False, set(), False, 30, 1., 1, lengths) all_attns = [] for i in range(min_length + 4): word_probs = torch.full( (batch_sz, n_words), -float('inf')) # "best" prediction is eos - that should be blocked word_probs[0, eos_idx] = valid_score_dist[0] # include at least one prediction OTHER than EOS # that is greater than -1e20 word_probs[0, _non_eos_idxs[0]] = valid_score_dist[1] word_probs[1:, _non_eos_idxs[0] + i] = 0 attns = torch.randn(1, batch_sz, 53) all_attns.append(attns) samp.advance(word_probs, attns) if i < min_length: self.assertTrue( samp.topk_scores[0].allclose(valid_score_dist[1])) self.assertTrue( samp.topk_scores[1:].eq(0).all()) elif i == min_length: # now batch 0 has ended and no others have self.assertTrue(samp.is_finished[0, :].eq(1).all()) self.assertTrue(samp.is_finished[1:, 1:].eq(0).all()) else: # i > min_length break
def test_advance_with_repeats_gets_blocked(self): n_words = 100 repeat_idx = 47 ngram_repeat = 3 for batch_sz in [1, 3]: samp = RandomSampling( 0, 1, 2, batch_sz, torch.device("cpu"), 0, ngram_repeat, set(), False, 30, 1., 5, torch.randint(0, 30, (batch_sz,))) for i in range(ngram_repeat + 4): # predict repeat_idx over and over again word_probs = torch.full( (batch_sz, n_words), -float('inf')) word_probs[:, repeat_idx] = 0 attns = torch.randn(1, batch_sz, 53) samp.advance(word_probs, attns) if i <= ngram_repeat: expected_scores = torch.zeros((batch_sz, 1)) self.assertTrue(samp.topk_scores.equal(expected_scores)) else: self.assertTrue( samp.topk_scores.equal( torch.tensor(self.BLOCKED_SCORE) .repeat(batch_sz, 1)))
def __getitem__(self, index): """ Args: index (int): Index Returns: tuple: (image, target) where target is class_index of the target class. """ # create random image that is consistent with the index id rng_state = torch.get_rng_state() torch.manual_seed(index + self.random_offset) img = torch.randn(*self.image_size) target = torch.randint(0, self.num_classes, size=(1,), dtype=torch.long)[0] torch.set_rng_state(rng_state) # convert to PIL Image img = transforms.ToPILImage()(img) if self.transform is not None: img = self.transform(img) if self.target_transform is not None: target = self.target_transform(target) return img, target
def train(model, start): # define Adam optimizer optimizer = optim.Adam(model.parameters(), lr=1e-6) # initialize mean squared error loss criterion = nn.MSELoss() # instantiate game game_state = GameState() # initialize replay memory replay_memory = [] # initial action is do nothing action = torch.zeros([model.number_of_actions], dtype=torch.float32) action[0] = 1 image_data, reward, terminal = game_state.frame_step(action) image_data = resize_and_bgr2gray(image_data) image_data = image_to_tensor(image_data) state = torch.cat((image_data, image_data, image_data, image_data)).unsqueeze(0) # initialize epsilon value epsilon = model.initial_epsilon iteration = 0 epsilon_decrements = np.linspace(model.initial_epsilon, model.final_epsilon, model.number_of_iterations) # main infinite loop while iteration < model.number_of_iterations: # get output from the neural network output = model(state)[0] # initialize action action = torch.zeros([model.number_of_actions], dtype=torch.float32) if torch.cuda.is_available(): # put on GPU if CUDA is available action = action.cuda() # epsilon greedy exploration random_action = random.random() <= epsilon if random_action: print("Performed random action!") action_index = [torch.randint(model.number_of_actions, torch.Size([]), dtype=torch.int) if random_action else torch.argmax(output)][0] if torch.cuda.is_available(): # put on GPU if CUDA is available action_index = action_index.cuda() action[action_index] = 1 # get next state and reward image_data_1, reward, terminal = game_state.frame_step(action) image_data_1 = resize_and_bgr2gray(image_data_1) image_data_1 = image_to_tensor(image_data_1) state_1 = torch.cat((state.squeeze(0)[1:, :, :], image_data_1)).unsqueeze(0) action = action.unsqueeze(0) reward = torch.from_numpy(np.array([reward], dtype=np.float32)).unsqueeze(0) # save transition to replay memory replay_memory.append((state, action, reward, state_1, terminal)) # if replay memory is full, remove the oldest transition if len(replay_memory) > model.replay_memory_size: replay_memory.pop(0) # epsilon annealing epsilon = epsilon_decrements[iteration] # sample random minibatch minibatch = random.sample(replay_memory, min(len(replay_memory), model.minibatch_size)) # unpack minibatch state_batch = torch.cat(tuple(d[0] for d in minibatch)) action_batch = torch.cat(tuple(d[1] for d in minibatch)) reward_batch = torch.cat(tuple(d[2] for d in minibatch)) state_1_batch = torch.cat(tuple(d[3] for d in minibatch)) if torch.cuda.is_available(): # put on GPU if CUDA is available state_batch = state_batch.cuda() action_batch = action_batch.cuda() reward_batch = reward_batch.cuda() state_1_batch = state_1_batch.cuda() # get output for the next state output_1_batch = model(state_1_batch) # set y_j to r_j for terminal state, otherwise to r_j + gamma*max(Q) y_batch = torch.cat(tuple(reward_batch[i] if minibatch[i][4] else reward_batch[i] + model.gamma * torch.max(output_1_batch[i]) for i in range(len(minibatch)))) # extract Q-value q_value = torch.sum(model(state_batch) * action_batch, dim=1) # PyTorch accumulates gradients by default, so they need to be reset in each pass optimizer.zero_grad() # returns a new Tensor, detached from the current graph, the result will never require gradient y_batch = y_batch.detach() # calculate loss loss = criterion(q_value, y_batch) # do backward pass loss.backward() optimizer.step() # set state to be state_1 state = state_1 iteration += 1 if iteration % 25000 == 0: torch.save(model, "pretrained_model/current_model_" + str(iteration) + ".pth") print("iteration:", iteration, "elapsed time:", time.time() - start, "epsilon:", epsilon, "action:", action_index.cpu().detach().numpy(), "reward:", reward.numpy()[0][0], "Q max:", np.max(output.cpu().detach().numpy()))
def NewOnesModuleFloat3D_basic(module, tu: TestUtils): module.forward(torch.randint(10, (2, 3)))
def test_param( self, degrees, translate, scale, shear, resample, align_corners, return_transform, same_on_batch, device, dtype ): _degrees = ( degrees if isinstance(degrees, (int, float, list, tuple)) else nn.Parameter(degrees.clone().to(device=device, dtype=dtype)) ) _translate = ( translate if isinstance(translate, (int, float, list, tuple)) else nn.Parameter(translate.clone().to(device=device, dtype=dtype)) ) _scale = ( scale if isinstance(scale, (int, float, list, tuple)) else nn.Parameter(scale.clone().to(device=device, dtype=dtype)) ) _shear = ( shear if isinstance(shear, (int, float, list, tuple)) else nn.Parameter(shear.clone().to(device=device, dtype=dtype)) ) torch.manual_seed(0) input = torch.randint(255, (2, 3, 10, 10), device=device, dtype=dtype) / 255.0 aug = RandomAffine( _degrees, _translate, _scale, _shear, resample, align_corners=align_corners, return_transform=return_transform, same_on_batch=same_on_batch, p=1.0, ) if return_transform: output, _ = aug(input) else: output = aug(input) if len(list(aug.parameters())) != 0: mse = nn.MSELoss() opt = torch.optim.SGD(aug.parameters(), lr=10) loss = mse(output, torch.ones_like(output) * 2) loss.backward() opt.step() if not isinstance(degrees, (int, float, list, tuple)): assert isinstance(aug.degrees, torch.Tensor) # Assert if param not updated if resample == 'nearest' and aug.degrees.is_cuda: # grid_sample in nearest mode and cuda device returns nan than 0 pass elif resample == 'nearest' or torch.all(aug.degrees._grad == 0.0): # grid_sample will return grad = 0 for resample nearest # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 assert (degrees.to(device=device, dtype=dtype) - aug.degrees.data).sum() == 0 else: assert (degrees.to(device=device, dtype=dtype) - aug.degrees.data).sum() != 0 if not isinstance(translate, (int, float, list, tuple)): assert isinstance(aug.translate, torch.Tensor) # Assert if param not updated if resample == 'nearest' and aug.translate.is_cuda: # grid_sample in nearest mode and cuda device returns nan than 0 pass elif resample == 'nearest' or torch.all(aug.translate._grad == 0.0): # grid_sample will return grad = 0 for resample nearest # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 assert (translate.to(device=device, dtype=dtype) - aug.translate.data).sum() == 0 else: assert (translate.to(device=device, dtype=dtype) - aug.translate.data).sum() != 0 if not isinstance(scale, (int, float, list, tuple)): assert isinstance(aug.scale, torch.Tensor) # Assert if param not updated if resample == 'nearest' and aug.scale.is_cuda: # grid_sample in nearest mode and cuda device returns nan than 0 pass elif resample == 'nearest' or torch.all(aug.scale._grad == 0.0): # grid_sample will return grad = 0 for resample nearest # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() == 0 else: assert (scale.to(device=device, dtype=dtype) - aug.scale.data).sum() != 0 if not isinstance(shear, (int, float, list, tuple)): assert isinstance(aug.shear, torch.Tensor) # Assert if param not updated if resample == 'nearest' and aug.shear.is_cuda: # grid_sample in nearest mode and cuda device returns nan than 0 pass elif resample == 'nearest' or torch.all(aug.shear._grad == 0.0): # grid_sample will return grad = 0 for resample nearest # https://discuss.pytorch.org/t/autograd-issue-with-f-grid-sample/76894 assert (shear.to(device=device, dtype=dtype) - aug.shear.data).sum() == 0 else: assert (shear.to(device=device, dtype=dtype) - aug.shear.data).sum() != 0
def FullLikeModuleInt2D_basic(module, tu: TestUtils): module.forward(torch.randint(10, (4, 5)))
def NewOnesModuleFalsePinMemory_basic(module, tu: TestUtils): module.forward(torch.randint(10, (2, 3)))
# -*- coding: utf-8 -*- import torch import numpy as np from Transformer import Transformer # Create random Tensors to hold inputs and outputs encoder_input = torch.randint(0, 100, (32, )) decoder_input = torch.tensor(np.arange(0, 15) + 1) decoder_output = torch.tensor(np.arange(0, 15) + 2) # Construct our model by instantiating the class defined above model = Transformer(vocab_size=100, embedding_size=256, max_seq_length=32) # Perform 100 training iterations. for i in range(40): model.fit(encoder_input, decoder_input, decoder_output) # Compare labels and predictions. print("\nenc inputs:\n", encoder_input) print("\ndec inputs:\n", decoder_input) print("\ndec labels:\n", decoder_output) print("\ndec prediction:\n", model.predict(encoder_input, decoder_input))
def test_ensemble_continuous_q_function( feature_size, action_size, batch_size, gamma, ensemble_size, q_func_type, n_quantiles, embed_size, bootstrap, use_independent_target, ): q_funcs = [] for _ in range(ensemble_size): encoder = DummyEncoder(feature_size, action_size, concat=True) if q_func_type == "mean": q_func = ContinuousMeanQFunction(encoder) elif q_func_type == "qr": q_func = ContinuousQRQFunction(encoder, n_quantiles) elif q_func_type == "iqn": q_func = ContinuousIQNQFunction(encoder, n_quantiles, n_quantiles, embed_size) elif q_func_type == "fqf": q_func = ContinuousFQFQFunction(encoder, n_quantiles, embed_size) q_funcs.append(q_func) q_func = EnsembleContinuousQFunction(q_funcs, bootstrap) # check output shape x = torch.rand(batch_size, feature_size) action = torch.rand(batch_size, action_size) values = q_func(x, action, "none") assert values.shape == (ensemble_size, batch_size, 1) # check compute_target target = q_func.compute_target(x, action) if q_func_type == "mean": assert target.shape == (batch_size, 1) min_values = values.min(dim=0).values assert (target == min_values).all() else: assert target.shape == (batch_size, n_quantiles) # check reductions if q_func_type != "iqn": assert torch.allclose(values.min(dim=0)[0], q_func(x, action, "min")) assert torch.allclose(values.max(dim=0)[0], q_func(x, action, "max")) assert torch.allclose(values.mean(dim=0), q_func(x, action, "mean")) # check td computation obs_t = torch.rand(batch_size, feature_size) act_t = torch.rand(batch_size, action_size) rew_tp1 = torch.rand(batch_size, 1) ter_tp1 = torch.randint(2, size=(batch_size, 1)) if q_func_type == "mean": if use_independent_target: q_tp1 = torch.rand(ensemble_size, batch_size, 1) else: q_tp1 = torch.rand(batch_size, 1) else: if use_independent_target: q_tp1 = torch.rand(ensemble_size, batch_size, n_quantiles) else: q_tp1 = torch.rand(batch_size, n_quantiles) ref_td_sum = 0.0 for i in range(ensemble_size): if use_independent_target: target = q_tp1[i] else: target = q_tp1 f = q_func.q_funcs[i] ref_td_sum += f.compute_error(obs_t, act_t, rew_tp1, target, ter_tp1, gamma) loss = q_func.compute_error(obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target) if bootstrap: assert not torch.allclose(ref_td_sum, loss) elif q_func_type != "iqn": assert torch.allclose(ref_td_sum, loss) # with mask mask = torch.rand(ensemble_size, batch_size, 1) loss = q_func.compute_error( obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target, mask, ) # check layer connection check_parameter_updates( q_func, (obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target), )
import torch from model import seq2seq, encoder, decoder voc_size = 10 hid_size = 50 sos = 0 eos = 1 max_len = 6 bidirectional = True batches, batch_size = 5, 10 # read data data = [ torch.randint(2, voc_size, size=(batch_size, max_len)) for _ in range(batches) ] targets = [ torch.randint(2, voc_size, size=(batch_size, max_len)) for _ in range(batches) ] # set model encoder = encoder.Encoder(voc_size=voc_size, hid_size=hid_size, bidirectional=bidirectional) decoder = decoder.Decoder(voc_size=voc_size, hid_size=hid_size, attn_type="dot-prod", max_len=max_len, sos=sos,
for word in input_words: try: idx = corpus.dictionary.word2idx[word] # if word is not in the vocabulary, exit the program and inform user except KeyError: sys.exit( f"'{word}' is not in the vocabulary. Please enter another word as input." ) # convert index to a tensor so it can be easily used as input. Then index is added to a list else: idx = torch.tensor(idx) idx_tensor = torch.reshape(idx, (1, 1)) input_idxs.append(idx_tensor) # if no input is given, still randomly produce the first word else: input_idxs = [torch.randint(ntokens, (1, 1), dtype=torch.long).to(device)] #print(corpus.dictionary.idx2word[input]) #print(input.shape) with open(args.outf, 'w') as outf: with torch.no_grad(): # no tracking history # My Addition: i = -1 # loop over the input words for input in input_idxs: i += 1 # write input word to file word = corpus.dictionary.idx2word[input] outf.write(word + ('\n' if i % 20 == 19 else ' ')) # give input word to hidden layer, output is not used (except in the last loop) output, hidden = model(input, hidden)
z = x + y print(z.stride()) # Ouputs: (3072, 1, 96, 3) ###################################################################### # Conv, Batchnorm modules using cudnn backends support channels last # (only works for CudNN >= 7.6). Convolution modules, unlike binary # p-wise operator, have channels last as the dominating memory format. # IFF all inputs are in contiguous memory format, the operator # produces output in contiguous memory format. Otherwise, output wil # be in channels last memroy format. if torch.backends.cudnn.version() >= 7603: model = torch.nn.Conv2d(8, 4, 3).cuda().half() model = model.to(memory_format=torch.channels_last) # Module parameters need to be channels last input = torch.randint(1, 10, (2, 8, 4, 4), dtype=torch.float32, requires_grad=True) input = input.to(device="cuda", memory_format=torch.channels_last, dtype=torch.float16) out = model(input) print(out.is_contiguous(memory_format=torch.channels_last)) # Ouputs: True ###################################################################### # When input tensor reaches a operator without channels last support, # a permutation should automatically apply in the kernel to restore # contiguous on input tensor. This introduces overhead and stops the # channels last memory format propagation. Nevertheless, it guarantees # correct output. ###################################################################### # Performance Gains # --------------------------------------------------------------------
acc = metrics.accuracy_score(labels_all, predict_all) return acc if __name__ == "__main__": debug = False # 相对路径 + modelName(TextCNN、TextLSTM) model_name = 'Transformer' module = import_module(model_name) config = module.Config(vocab_size, embed_dim, label_num) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = module.Model(config).to(device) if debug: # 维度:batch_size * max_length, 数值:0~200之间的整数,每一行表示wordid inputs = torch.randint(0, 200, (batch_size, max_length)) # 维度:batch_size * 1, 数值:0~2之间的整数,维度扩充1,和input对应 labels = torch.randint(0, 2, (batch_size, 1)).squeeze(0) print(model(inputs)) else: x_train, y_train, label_num = get_data(train_path) dataset = DealDataset(x_train, y_train, device) dataloader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True) x_dev, y_dev, _ = get_data(dev_path) dataset_dev = DealDataset(x_dev, y_dev, device) dataloader_dev = DataLoader(dataset=dataset_dev, batch_size=batch_size, shuffle=True)
def __getitem__(self, i): x, y = self.dataset[i] if torch.randn(1).item() < 0: return x, 1, y return x, 0, torch.randint(0, 10, (1, )).item()
def get_test_cases(): test_cases = [ (torch.randint(0, 2, size=(10, 4)).long(), torch.randint(0, 2, size=(10, 4)).long(), 1), (torch.randint(0, 2, size=(100, 7)).long(), torch.randint(0, 2, size=(100, 7)).long(), 1), (torch.randint(0, 2, size=(100, 5)).long(), torch.randint(0, 2, size=(100, 5)).long(), 1), (torch.randint(0, 2, size=(100, 3)).long(), torch.randint(0, 2, size=(100, 3)).long(), 1), # updated batches (torch.randint(0, 2, size=(10, 4)).long(), torch.randint(0, 2, size=(10, 4)).long(), 16), (torch.randint(0, 2, size=(100, 7)).long(), torch.randint(0, 2, size=(100, 7)).long(), 16), (torch.randint(0, 2, size=(100, 5)).long(), torch.randint(0, 2, size=(100, 5)).long(), 16), (torch.randint(0, 2, size=(100, 3)).long(), torch.randint(0, 2, size=(100, 3)).long(), 16), ] return test_cases
import torch import numpy as np import torchvision.transforms.functional as TF mean = [0.5, 0.5, 0.5] var = [0.5, 0.5, 0.5] ''' map = Image.open('2.tif') print("original map") map_array = np.asarray(map) print(map_array.shape) print(map_array.dtype) ''' map = torch.randint(low=0, high=255, size=(3, 3, 3), dtype=torch.uint8) map_array = np.asarray(map) print(map_array) to_tensor_map = TF.to_tensor(map_array) print("scaled map") print(to_tensor_map.shape) print(to_tensor_map) normal_map = TF.normalize(to_tensor_map, mean, var) print("normal_map") print(normal_map) # manaul_norm mean = torch.mean(to_tensor_map, dim=(0, 1)) var = torch.var(to_tensor_map, dim=2, keepdim=False, unbiased=False) manaul_norm_map = (to_tensor_map - mean) / var print(manaul_norm_map)
def test_show_result_meshlab(): pcd = 'tests/data/nuscenes/samples/LIDAR_TOP/n015-2018-08-02-17-16-37+' \ '0800__LIDAR_TOP__1533201470948018.pcd.bin' box_3d = LiDARInstance3DBoxes( torch.tensor( [[8.7314, -1.8559, -1.5997, 0.4800, 1.2000, 1.8900, 0.0100]])) labels_3d = torch.tensor([0]) scores_3d = torch.tensor([0.5]) points = np.random.rand(100, 4) img_meta = dict(pts_filename=pcd, boxes_3d=box_3d, box_mode_3d=Box3DMode.LIDAR) data = dict(points=[[torch.tensor(points)]], img_metas=[[img_meta]]) result = [ dict(pts_bbox=dict( boxes_3d=box_3d, labels_3d=labels_3d, scores_3d=scores_3d)) ] tmp_dir = tempfile.TemporaryDirectory() temp_out_dir = tmp_dir.name out_dir, file_name = show_result_meshlab(data, result, temp_out_dir) expected_outfile_pred = file_name + '_pred.obj' expected_outfile_pts = file_name + '_points.obj' expected_outfile_pred_path = os.path.join(out_dir, file_name, expected_outfile_pred) expected_outfile_pts_path = os.path.join(out_dir, file_name, expected_outfile_pts) assert os.path.exists(expected_outfile_pred_path) assert os.path.exists(expected_outfile_pts_path) tmp_dir.cleanup() # test multi-modality show # indoor scene pcd = 'tests/data/sunrgbd/points/000001.bin' filename = 'tests/data/sunrgbd/sunrgbd_trainval/image/000001.jpg' box_3d = DepthInstance3DBoxes( torch.tensor( [[-1.1580, 3.3041, -0.9961, 0.3829, 0.4647, 0.5574, 1.1213]])) img = np.random.randn(1, 3, 608, 832) k_mat = np.array([[529.5000, 0.0000, 365.0000], [0.0000, 529.5000, 265.0000], [0.0000, 0.0000, 1.0000]]) rt_mat = np.array([[0.9980, 0.0058, -0.0634], [0.0058, 0.9835, 0.1808], [0.0634, -0.1808, 0.9815]]) rt_mat = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) @ rt_mat.transpose( 1, 0) depth2img = k_mat @ rt_mat img_meta = dict(filename=filename, depth2img=depth2img, pcd_horizontal_flip=False, pcd_vertical_flip=False, box_mode_3d=Box3DMode.DEPTH, box_type_3d=DepthInstance3DBoxes, pcd_trans=np.array([0., 0., 0.]), pcd_scale_factor=1.0, pts_filename=pcd, transformation_3d_flow=['R', 'S', 'T']) data = dict(points=[[torch.tensor(points)]], img_metas=[[img_meta]], img=[img]) result = [dict(boxes_3d=box_3d, labels_3d=labels_3d, scores_3d=scores_3d)] tmp_dir = tempfile.TemporaryDirectory() temp_out_dir = tmp_dir.name out_dir, file_name = show_result_meshlab(data, result, temp_out_dir, 0.3, task='multi_modality-det') expected_outfile_pred = file_name + '_pred.obj' expected_outfile_pts = file_name + '_points.obj' expected_outfile_png = file_name + '_img.png' expected_outfile_proj = file_name + '_pred.png' expected_outfile_pred_path = os.path.join(out_dir, file_name, expected_outfile_pred) expected_outfile_pts_path = os.path.join(out_dir, file_name, expected_outfile_pts) expected_outfile_png_path = os.path.join(out_dir, file_name, expected_outfile_png) expected_outfile_proj_path = os.path.join(out_dir, file_name, expected_outfile_proj) assert os.path.exists(expected_outfile_pred_path) assert os.path.exists(expected_outfile_pts_path) assert os.path.exists(expected_outfile_png_path) assert os.path.exists(expected_outfile_proj_path) tmp_dir.cleanup() # outdoor scene pcd = 'tests/data/kitti/training/velodyne_reduced/000000.bin' filename = 'tests/data/kitti/training/image_2/000000.png' box_3d = LiDARInstance3DBoxes( torch.tensor( [[6.4495, -3.9097, -1.7409, 1.5063, 3.1819, 1.4716, 1.8782]])) img = np.random.randn(1, 3, 384, 1280) lidar2img = np.array( [[6.09695435e+02, -7.21421631e+02, -1.25125790e+00, -1.23041824e+02], [1.80384201e+02, 7.64479828e+00, -7.19651550e+02, -1.01016693e+02], [9.99945343e-01, 1.24365499e-04, 1.04513029e-02, -2.69386917e-01], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.00000000e+00]]) img_meta = dict(filename=filename, pcd_horizontal_flip=False, pcd_vertical_flip=False, box_mode_3d=Box3DMode.LIDAR, box_type_3d=LiDARInstance3DBoxes, pcd_trans=np.array([0., 0., 0.]), pcd_scale_factor=1.0, pts_filename=pcd, lidar2img=lidar2img) data = dict(points=[[torch.tensor(points)]], img_metas=[[img_meta]], img=[img]) result = [ dict(pts_bbox=dict( boxes_3d=box_3d, labels_3d=labels_3d, scores_3d=scores_3d)) ] out_dir, file_name = show_result_meshlab(data, result, temp_out_dir, 0.1, task='multi_modality-det') tmp_dir = tempfile.TemporaryDirectory() temp_out_dir = tmp_dir.name expected_outfile_pred = file_name + '_pred.obj' expected_outfile_pts = file_name + '_points.obj' expected_outfile_png = file_name + '_img.png' expected_outfile_proj = file_name + '_pred.png' expected_outfile_pred_path = os.path.join(out_dir, file_name, expected_outfile_pred) expected_outfile_pts_path = os.path.join(out_dir, file_name, expected_outfile_pts) expected_outfile_png_path = os.path.join(out_dir, file_name, expected_outfile_png) expected_outfile_proj_path = os.path.join(out_dir, file_name, expected_outfile_proj) assert os.path.exists(expected_outfile_pred_path) assert os.path.exists(expected_outfile_pts_path) assert os.path.exists(expected_outfile_png_path) assert os.path.exists(expected_outfile_proj_path) tmp_dir.cleanup() # test mono-3d show filename = 'tests/data/nuscenes/samples/CAM_BACK_LEFT/n015-2018-' \ '07-18-11-07-57+0800__CAM_BACK_LEFT__1531883530447423.jpg' box_3d = CameraInstance3DBoxes( torch.tensor( [[6.4495, -3.9097, -1.7409, 1.5063, 3.1819, 1.4716, 1.8782]])) img = np.random.randn(1, 3, 384, 1280) cam_intrinsic = np.array([[100.0, 0.0, 50.0], [0.0, 100.0, 50.0], [0.0, 0.0, 1.0]]) img_meta = dict(filename=filename, pcd_horizontal_flip=False, pcd_vertical_flip=False, box_mode_3d=Box3DMode.CAM, box_type_3d=CameraInstance3DBoxes, pcd_trans=np.array([0., 0., 0.]), pcd_scale_factor=1.0, cam_intrinsic=cam_intrinsic) data = dict(points=[[torch.tensor(points)]], img_metas=[[img_meta]], img=[img]) result = [ dict(img_bbox=dict( boxes_3d=box_3d, labels_3d=labels_3d, scores_3d=scores_3d)) ] out_dir, file_name = show_result_meshlab(data, result, temp_out_dir, 0.1, task='mono-det') tmp_dir = tempfile.TemporaryDirectory() temp_out_dir = tmp_dir.name expected_outfile_png = file_name + '_img.png' expected_outfile_proj = file_name + '_pred.png' expected_outfile_png_path = os.path.join(out_dir, file_name, expected_outfile_png) expected_outfile_proj_path = os.path.join(out_dir, file_name, expected_outfile_proj) assert os.path.exists(expected_outfile_png_path) assert os.path.exists(expected_outfile_proj_path) tmp_dir.cleanup() # test seg show pcd = 'tests/data/scannet/points/scene0000_00.bin' points = np.random.rand(100, 6) img_meta = dict(pts_filename=pcd) data = dict(points=[[torch.tensor(points)]], img_metas=[[img_meta]]) pred_seg = torch.randint(0, 20, (100, )) result = [dict(semantic_mask=pred_seg)] tmp_dir = tempfile.TemporaryDirectory() temp_out_dir = tmp_dir.name out_dir, file_name = show_result_meshlab(data, result, temp_out_dir, task='seg') expected_outfile_pred = file_name + '_pred.obj' expected_outfile_pts = file_name + '_points.obj' expected_outfile_pred_path = os.path.join(out_dir, file_name, expected_outfile_pred) expected_outfile_pts_path = os.path.join(out_dir, file_name, expected_outfile_pts) assert os.path.exists(expected_outfile_pred_path) assert os.path.exists(expected_outfile_pts_path) tmp_dir.cleanup()
def presgan(dat, netG, netD, log_sigma, args): writer = SummaryWriter(log_dir='tensorboard' + args.dataset) device = args.device if torch.cuda.is_available(): print("cuda") netG.cuda() netD.cuda() criterion.cuda() criterion_mse.cuda() X_training = dat['X_train'].to(device) # [60000, 1, 64, 64] fixed_noise = torch.randn(args.num_gen_images, args.nz, 1, 1, device=device) torch.manual_seed(123) # NEW Y_training = dat['Y_train'].to(device) # NUM_CLASS = 10 NUM_CLASS = args.n_classes optimizerD = optim.Adam(netD.parameters(), lr=args.lrD, betas=(args.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=args.lrG, betas=(args.beta1, 0.999)) sigma_optimizer = optim.Adam([log_sigma], lr=args.sigma_lr, betas=(args.beta1, 0.999)) if args.restrict_sigma: logsigma_min = math.log(math.exp(args.sigma_min) - 1.0) logsigma_max = math.log(math.exp(args.sigma_max) - 1.0) #stepsize = args.stepsize_num / args.nz #bsz = args.batchSize #print(X_training.shape) #print(X_training.shape) #print(X_training.shape) #asdfasdfcscv stepsize = args.stepsize_num / args.nz Y_forY_training = dat['Y_train'].to(device) bsz = args.batchSize for epoch in range(1, args.epochs + 1): for i in range(0, len(X_training), bsz): # sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) # sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) # sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) stop = min(bsz, len(X_training[i:])) real_cpu = X_training[i:i + stop].to(device) y_real_cpu = Y_forY_training[i:i + stop].to(device) for sadgfjasj in range(len(y_real_cpu)): if (sadgfjasj > 0) and (y_real_cpu[sadgfjasj] == 2): y_real_cpu[sadgfjasj] = y_real_cpu[sadgfjasj - 1] real_cpu[sadgfjasj, :] = real_cpu[sadgfjasj - 1, :] elif (sadgfjasj == 0) and (y_real_cpu[sadgfjasj] == 2): y_real_cpu[sadgfjasj] = y_real_cpu[sadgfjasj + 1] real_cpu[sadgfjasj, :] = real_cpu[sadgfjasj + 1, :] X_training[i:i + stop] = real_cpu Y_forY_training[i:i + stop] = y_real_cpu #sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) #sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) #sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) #sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) netD.zero_grad() stop = min(bsz, len(X_training[i:])) real_cpu = X_training[i:i + stop].to(device) ''' for epoch in range(1, args.epochs+1): for i in range(0, len(X_training), bsz): # bsz = 64 sigma_x = F.softplus(log_sigma).view(1, 1, args.imageSize, args.imageSize) netD.zero_grad() stop = min(bsz, len(X_training[i:])) real_cpu = X_training[i:i+stop].to(device) # [64, 1, 64, 64] ''' batch_size = real_cpu.size(0) labelv = torch.full((batch_size, ), real_label).to(device) # train discriminator on real (noised) data and real labels y_labels = Y_training[i:i + stop].to(device) y_one_hot = torch.FloatTensor(batch_size, NUM_CLASS).to( device) # adding cuda here # print(batch_size, bsz, y_labels.size()) y_one_hot = y_one_hot.zero_().scatter_( 1, y_labels.view(batch_size, 1), 1).to(device) noise_eta = torch.randn_like(real_cpu).to(device) noised_data = real_cpu + sigma_x.detach() * noise_eta out_real = netD(noised_data, y_one_hot) #, y_one_hot_labels errD_real = criterion(out_real, labelv) errD_real.backward() D_x = out_real.mean().item() # make generator output image from random labels; make discriminator classify rand_y_one_hot = torch.FloatTensor( batch_size, NUM_CLASS).zero_().to(device) # adding cuda here rand_y_one_hot.scatter_( 1, torch.randint(0, NUM_CLASS, size=(batch_size, 1), device=device), 1 ) # #rand_y_one_hot.scatter_(1, torch.from_numpy(np.random.randint(0, 10, size=(bsz,1))), 1) noise = torch.randn(batch_size, args.nz, 1, 1, device=device) mu_fake = netG(noise, rand_y_one_hot) fake = mu_fake + sigma_x * noise_eta labelv = labelv.fill_(fake_label).to(device) out_fake = netD(fake.detach(), rand_y_one_hot) errD_fake = criterion(out_fake, labelv) errD_fake.backward() D_G_z1 = out_fake.mean().item() errD = errD_real + errD_fake optimizerD.step() # update G network: maximize log(D(G(z))) netG.zero_grad() sigma_optimizer.zero_grad() rand_y_one_hot = torch.FloatTensor(batch_size, NUM_CLASS).zero_().to(device) rand_y_one_hot = rand_y_one_hot.scatter_( 1, torch.randint(0, NUM_CLASS, size=(batch_size, 1), device=device), 1).to(device) labelv = labelv.fill_(real_label).to(device) gen_input = torch.randn(batch_size, args.nz, 1, 1, device=device) out = netG(gen_input, rand_y_one_hot) # add rand y labels noise_eta = torch.randn_like(out) g_fake_data = out + noise_eta * sigma_x dg_fake_decision = netD(g_fake_data, rand_y_one_hot) # add rand y labels g_error_gan = criterion(dg_fake_decision, labelv) D_G_z2 = dg_fake_decision.mean().item() # # TO TEST WITHOUT ENTROPY, SET: # if epoch < 10 and args.lambda_ != 0 and args.dataset != 'mnist': # args.lambda_ = 0 # elif epoch < 20 and args.lambda_ != 0 and args.dataset != 'mnist': # args.lambda_ = 0.0001 # elif args.lambda_ != 0 and args.dataset != 'mnist': # args.lambda_ = 0.0002 if args.lambda_ == 0: g_error_gan.backward() optimizerG.step() sigma_optimizer.step() else: # added y_tilde param (rand_y_one_hot) hmc_samples, hmc_labels, acceptRate, stepsize = hmc.get_samples( netG, g_fake_data.detach(), rand_y_one_hot.detach(), gen_input.clone(), sigma_x.detach(), args.burn_in, args.num_samples_posterior, args.leapfrog_steps, stepsize, args.flag_adapt, args.hmc_learning_rate, args.hmc_opt_accept) bsz, d = hmc_samples.size() hmc_samples = hmc_samples.view(bsz, d, 1, 1).to(device) hmc_labels = hmc_labels.to(device) mean_output = netG(hmc_samples, hmc_labels) bsz = g_fake_data.size(0) mean_output_summed = torch.zeros_like(g_fake_data).to(device) for cnt in range(args.num_samples_posterior): mean_output_summed = mean_output_summed + mean_output[ cnt * bsz:(cnt + 1) * bsz] mean_output_summed = mean_output_summed / args.num_samples_posterior c = ((g_fake_data - mean_output_summed) / sigma_x**2).detach() g_error_entropy = torch.mul(c, out + sigma_x * noise_eta).mean(0).sum() g_error = g_error_gan - args.lambda_ * g_error_entropy g_error.backward() optimizerG.step() sigma_optimizer.step() if args.restrict_sigma: log_sigma.data.clamp_(min=logsigma_min, max=logsigma_max) ## log performance if i % args.log == 0: print( 'Epoch [%d/%d] .. Batch [%d/%d] .. Loss_D: %.4f .. Loss_G: %.4f .. D(x): %.4f .. D(G(z)): %.4f / %.4f' % (epoch, args.epochs, i, len(X_training), errD.data, g_error_gan.data, D_x, D_G_z1, D_G_z2)) with open('%s/log.csv' % args.results_folder, 'a') as f: r = csv.writer(f) # Loss_G, Loss_D, D(x), D(G(z)) r.writerow([g_error_gan.data, errD.data, D_x, D_G_z2]) if i % (2 * args.log) == 0: t_iter = (epoch * len(X_training) + i) / bsz writer.add_scalar('Loss_G', g_error_gan.data, t_iter) writer.add_scalar('Loss_D', errD.data, t_iter) writer.add_scalar('D(x)', D_x, t_iter) writer.add_scalar('D(G(z))', D_G_z2, t_iter) print('*' * 100) print('End of epoch {}'.format(epoch)) print('sigma min: {} .. sigma max: {}'.format(torch.min(sigma_x), torch.max(sigma_x))) print('*' * 100) if args.lambda_ > 0: print( '| MCMC diagnostics ====> | stepsize: {} | min ar: {} | mean ar: {} | max ar: {} |' .format(stepsize, acceptRate.min().item(), acceptRate.mean().item(), acceptRate.max().item())) if epoch % args.save_imgs_every == 0: rand_y_one_hot = torch.FloatTensor(args.num_gen_images, NUM_CLASS).zero_().to( device) # adding cuda here rand_y_one_hot = rand_y_one_hot.scatter_( 1, torch.randint(0, NUM_CLASS, size=(args.num_gen_images, 1), device=device), 1 ).to( device ) # #rand_y_one_hot.scatter_(1, torch.from_numpy(np.random.randint(0, 10, size=(bsz,1))), 1) fake = netG(fixed_noise, rand_y_one_hot).detach() vutils.save_image(fake, '%s/presgan_%s_fake_epoch_%03d.png' % (args.results_folder, args.dataset, epoch), normalize=True, nrow=20) if epoch % args.save_ckpt_every == 0: torch.save( netG.state_dict(), os.path.join( args.results_folder, 'netG_presgan_%s_epoch_%s.pth' % (args.dataset, epoch))) torch.save( log_sigma, os.path.join(args.results_folder, 'log_sigma_%s_%s.pth' % (args.dataset, epoch))) torch.save( netD.state_dict(), os.path.join( args.results_folder, 'netD_presgan_%s_epoch_%s.pth' % (args.dataset, epoch)))
def test_ensemble_discrete_q_function( feature_size, action_size, batch_size, gamma, ensemble_size, q_func_type, n_quantiles, embed_size, bootstrap, use_independent_target, ): q_funcs = [] for _ in range(ensemble_size): encoder = DummyEncoder(feature_size) if q_func_type == "mean": q_func = DiscreteMeanQFunction(encoder, action_size) elif q_func_type == "qr": q_func = DiscreteQRQFunction(encoder, action_size, n_quantiles) elif q_func_type == "iqn": q_func = DiscreteIQNQFunction(encoder, action_size, n_quantiles, n_quantiles, embed_size) elif q_func_type == "fqf": q_func = DiscreteFQFQFunction(encoder, action_size, n_quantiles, embed_size) q_funcs.append(q_func) q_func = EnsembleDiscreteQFunction(q_funcs, bootstrap) # check output shape x = torch.rand(batch_size, feature_size) values = q_func(x, "none") assert values.shape == (ensemble_size, batch_size, action_size) # check compute_target action = torch.randint(high=action_size, size=(batch_size, )) target = q_func.compute_target(x, action) if q_func_type == "mean": assert target.shape == (batch_size, 1) min_values = values.min(dim=0).values assert torch.allclose(min_values[torch.arange(batch_size), action], target.view(-1)) else: assert target.shape == (batch_size, n_quantiles) # check compute_target with action=None targets = q_func.compute_target(x) if q_func_type == "mean": assert targets.shape == (batch_size, action_size) else: assert targets.shape == (batch_size, action_size, n_quantiles) # check reductions if q_func_type != "iqn": assert torch.allclose(values.min(dim=0).values, q_func(x, "min")) assert torch.allclose(values.max(dim=0).values, q_func(x, "max")) assert torch.allclose(values.mean(dim=0), q_func(x, "mean")) # check td computation obs_t = torch.rand(batch_size, feature_size) act_t = torch.randint(0, action_size, size=(batch_size, 1), dtype=torch.int64) rew_tp1 = torch.rand(batch_size, 1) ter_tp1 = torch.randint(2, size=(batch_size, 1)) if q_func_type == "mean": if use_independent_target: q_tp1 = torch.rand(ensemble_size, batch_size, 1) else: q_tp1 = torch.rand(batch_size, 1) else: if use_independent_target: q_tp1 = torch.rand(ensemble_size, batch_size, n_quantiles) else: q_tp1 = torch.rand(batch_size, n_quantiles) ref_td_sum = 0.0 for i in range(ensemble_size): f = q_func.q_funcs[i] if use_independent_target: target = q_tp1[i] else: target = q_tp1 ref_td_sum += f.compute_error(obs_t, act_t, rew_tp1, target, ter_tp1, gamma) loss = q_func.compute_error(obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target) if bootstrap: assert not torch.allclose(ref_td_sum, loss) elif q_func_type != "iqn": assert torch.allclose(ref_td_sum, loss) # with mask mask = torch.rand(ensemble_size, batch_size, 1) loss = q_func.compute_error( obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target, mask, ) # check layer connection check_parameter_updates( q_func, (obs_t, act_t, rew_tp1, q_tp1, ter_tp1, gamma, use_independent_target), )
def reconstruct(self): self.G.eval() self.load() # for training (partition the training data in case memory overflow) torch.manual_seed(self.manual_seed) for k in range(self.train_parts): sample_z = torch.rand((self.train_size, self.z_dim)) labels = torch.randint(0, self.class_num, (self.train_size, 1)).type(torch.LongTensor) sample_y = torch.zeros(self.train_size, self.class_num).scatter_(1, labels, 1) if self.gpu_mode: sample_z, sample_y = sample_z.cuda(), sample_y.cuda() samples = (self.G(sample_z, sample_y) + 1) / 2 if self.gpu_mode: samples = samples.cpu().data.numpy() else: samples = samples.data.numpy() if k == 0: labels_train = labels samples_train = samples else: labels_train = np.concatenate((labels_train, labels), axis=0) samples_train = np.concatenate((samples_train, samples), axis=0) data_dir = 'data/' + self.dataset + '/' + self.model_name if not os.path.exists(data_dir): os.makedirs(data_dir) np.savez(data_dir + '/train', sample=samples_train, label=labels_train.squeeze(1)) # for testing torch.manual_seed(self.manual_seed + 999) for k in range(self.test_parts): sample_z = torch.rand((self.test_size, self.z_dim)) labels = torch.randint(0, self.class_num, (self.test_size, 1)).type(torch.LongTensor) sample_y = torch.zeros(self.test_size, self.class_num).scatter_(1, labels, 1) if self.gpu_mode: sample_z, sample_y = sample_z.cuda(), sample_y.cuda() samples = (self.G(sample_z, sample_y) + 1) / 2 if self.gpu_mode: samples = samples.cpu().data.numpy() else: samples = samples.data.numpy() if k == 0: labels_test = labels samples_test = samples else: labels_test = np.concatenate((labels_test, labels), axis=0) samples_test = np.concatenate((samples_test, samples), axis=0) np.savez(data_dir + '/test', sample=samples_test, label=labels_test.squeeze(1)) samples_test = samples_test.transpose(0, 2, 3, 1) # compute the local lipschitz constant of a generator using samples utils.save_images( samples_test[:100, :, :, :], [10, 10], self.save_dir + '/' + self.dataset + '/' + self.model_name + '/gen_img.png')
def test_beam_returns_attn_with_correct_length(self): beam_sz = 5 batch_sz = 3 n_words = 100 _non_eos_idxs = [47, 51, 13, 88, 99] valid_score_dist = torch.log_softmax(torch.tensor( [6., 5., 4., 3., 2., 1.]), dim=0) min_length = 5 eos_idx = 2 inp_lens = torch.randint(1, 30, (batch_sz,)) beam = BeamSearch( beam_sz, batch_sz, 0, 1, 2, 2, torch.device("cpu"), GlobalScorerStub(), min_length, 30, True, 0, set(), inp_lens, False, 0.) for i in range(min_length + 2): # non-interesting beams are going to get dummy values word_probs = torch.full( (batch_sz * beam_sz, n_words), -float('inf')) if i == 0: # "best" prediction is eos - that should be blocked word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] # include at least beam_sz predictions OTHER than EOS # that are greater than -1e20 for j, score in zip(_non_eos_idxs, valid_score_dist[1:]): word_probs[0::beam_sz, j] = score elif i <= min_length: # predict eos in beam 1 word_probs[1::beam_sz, eos_idx] = valid_score_dist[0] # provide beam_sz other good predictions in other beams for k, (j, score) in enumerate( zip(_non_eos_idxs, valid_score_dist[1:])): beam_idx = min(beam_sz-1, k) word_probs[beam_idx::beam_sz, j] = score else: word_probs[0::beam_sz, eos_idx] = valid_score_dist[0] word_probs[1::beam_sz, eos_idx] = valid_score_dist[0] # provide beam_sz other good predictions in other beams for k, (j, score) in enumerate( zip(_non_eos_idxs, valid_score_dist[1:])): beam_idx = min(beam_sz-1, k) word_probs[beam_idx::beam_sz, j] = score attns = torch.randn(1, batch_sz * beam_sz, 53) beam.advance(word_probs, attns) if i < min_length: self.assertFalse(beam.done) # no top beams are finished yet for b in range(batch_sz): self.assertEqual(beam.attention[b], []) elif i == min_length: # beam 1 dies on min_length self.assertTrue(beam.is_finished[:, 1].all()) beam.update_finished() self.assertFalse(beam.done) # no top beams are finished yet for b in range(batch_sz): self.assertEqual(beam.attention[b], []) else: # i > min_length # beam 0 dies on the step after beam 1 dies self.assertTrue(beam.is_finished[:, 0].all()) beam.update_finished() self.assertTrue(beam.done) # top beam is finished now so there are attentions for b in range(batch_sz): # two beams are finished in each batch self.assertEqual(len(beam.attention[b]), 2) for k in range(2): # second dim is cut down to the non-padded src length self.assertEqual(beam.attention[b][k].shape[-1], inp_lens[b]) # first dim is equal to the time of death # (beam 0 died at current step - adjust for SOS) self.assertEqual(beam.attention[b][0].shape[0], i+1) # (beam 1 died at last step - adjust for SOS) self.assertEqual(beam.attention[b][1].shape[0], i) # behavior gets weird when beam is already done so just stop break
# summing BF1 Score for each class and average over mini-batch loss = torch.mean(1 - BF1) return loss # for debug if __name__ == "__main__": import torch.optim as optim from torchvision.models import segmentation device = 'cuda' if torch.cuda.is_available() else 'cpu' img = torch.randn(8, 3, 224, 224).to(device) gt = torch.randint(0, 10, (8, 224, 224)).to(device) model = segmentation.fcn_resnet50(num_classes=10).to(device) optimizer = optim.Adam(model.parameters(), lr=0.0001) criterion = BoundaryLoss() y = model(img) loss = criterion(y['out'], gt) optimizer.zero_grad() loss.backward() optimizer.step() print(loss)
download=True) print(train_data) print(test_data) print(train_data.data.size()) print(train_data.targets.size()) print(test_data.data.size()) print(test_data.targets.size()) #plot some of the train data figure = plt.figure(figsize=(10, 8)) plt.title('Random Images from MNIST Data') plt.axis("off") cols, rows = 10, 10 for i in range(1, cols * rows + 1): sample_idx = torch.randint(len(train_data.data), size=(1, )).item() img = train_data.data[sample_idx] figure.add_subplot(rows, cols, i) plt.axis("off") plt.imshow(img.squeeze(), cmap="gray") plt.show() sequence_length = 28 input_size = 28 hidden_size1 = 32 hidden_size2 = 64 hidden_size3 = 128 hidden_size4 = 256 num_layers = 1 batch_size = 100 num_epochs = 50
# That's it. We can now pass ``extension`` to a # :py:class:`with backpack(...) <backpack.backpack>` context and compute individual # gradients with respect to ``ScaleModule``'s ``weight`` parameter. # %% # Test custom module # ------------------ # Here, we verify the custom module extension on a small net with random inputs. # Let's create these. batch_size = 10 batch_axis = 0 input_size = 4 inputs = torch.randn(batch_size, input_size, device=device) targets = torch.randint(0, 2, (batch_size, ), device=device) reduction = ["mean", "sum"][1] my_module = ScaleModule().to(device) lossfunc = torch.nn.CrossEntropyLoss(reduction=reduction).to(device) # %% # .. note:: # Results of ``"mean"`` and ``"sum"`` reduction differ by a scaling factor, # because the information backpropagated by PyTorch is scaled. This is documented at # https://docs.backpack.pt/en/master/extensions.html#backpack.extensions.BatchGrad. # %% # Individual gradients with PyTorch # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # The following computes individual gradients by looping over individual samples and
res0 = x x = self.downsample1(x) print(x.shape) res1 = x x = self.downsample2(x) print(x.shape) res2 = x x = self.downsample3(x) print(x.shape) x = self.upsample1(x) x = res2 + x x = self.upsample2(x) x = res1 + x x = self.upsample3(x) x = res0 + x x = self.output(x) return x if __name__ == "__main__": x = torch.randn(1, 4, 64, 64, 64) Input = torch.randn(1, 2, 6, 128, 128) label = torch.randint(0, 4, (1, 10, 128, 128)) model = VFN() out = model(Input) # print(model) print(out.shape) # print(label.shape) # out = torch.argmax(out, dim=1).float() # print(out)
def assign_targets_single(self, anchors, gt_boxes, gt_classes, matched_threshold=0.6, unmatched_threshold=0.45): num_anchors = anchors.shape[0] num_gt = gt_boxes.shape[0] labels = torch.ones( (num_anchors, ), dtype=torch.int32, device=anchors.device) * -1 gt_ids = torch.ones( (num_anchors, ), dtype=torch.int32, device=anchors.device) * -1 if len(gt_boxes) > 0 and anchors.shape[0] > 0: anchor_by_gt_overlap = iou3d_nms_utils.boxes_iou3d_gpu(anchors[:, 0:7], gt_boxes[:, 0:7]) \ if self.match_height else box_utils.boxes3d_nearest_bev_iou(anchors[:, 0:7], gt_boxes[:, 0:7]) anchor_to_gt_argmax = torch.from_numpy( anchor_by_gt_overlap.cpu().numpy().argmax(axis=1)).cuda() anchor_to_gt_max = anchor_by_gt_overlap[ torch.arange(num_anchors, device=anchors.device), anchor_to_gt_argmax] gt_to_anchor_argmax = torch.from_numpy( anchor_by_gt_overlap.cpu().numpy().argmax(axis=0)).cuda() gt_to_anchor_max = anchor_by_gt_overlap[ gt_to_anchor_argmax, torch.arange(num_gt, device=anchors.device)] empty_gt_mask = gt_to_anchor_max == 0 gt_to_anchor_max[empty_gt_mask] = -1 anchors_with_max_overlap = ( anchor_by_gt_overlap == gt_to_anchor_max).nonzero()[:, 0] gt_inds_force = anchor_to_gt_argmax[anchors_with_max_overlap] labels[anchors_with_max_overlap] = gt_classes[gt_inds_force] gt_ids[anchors_with_max_overlap] = gt_inds_force.int() pos_inds = anchor_to_gt_max >= matched_threshold gt_inds_over_thresh = anchor_to_gt_argmax[pos_inds] labels[pos_inds] = gt_classes[gt_inds_over_thresh] gt_ids[pos_inds] = gt_inds_over_thresh.int() bg_inds = (anchor_to_gt_max < unmatched_threshold).nonzero()[:, 0] else: bg_inds = torch.arange(num_anchors, device=anchors.device) fg_inds = (labels > 0).nonzero()[:, 0] if self.pos_fraction is not None: num_fg = int(self.pos_fraction * self.sample_size) if len(fg_inds) > num_fg: num_disabled = len(fg_inds) - num_fg disable_inds = torch.randperm(len(fg_inds))[:num_disabled] labels[disable_inds] = -1 fg_inds = (labels > 0).nonzero()[:, 0] num_bg = self.sample_size - (labels > 0).sum() if len(bg_inds) > num_bg: enable_inds = bg_inds[torch.randint(0, len(bg_inds), size=(num_bg, ))] labels[enable_inds] = 0 # bg_inds = torch.nonzero(labels == 0)[:, 0] else: if len(gt_boxes) == 0 or anchors.shape[0] == 0: labels[:] = 0 else: labels[bg_inds] = 0 labels[anchors_with_max_overlap] = gt_classes[gt_inds_force] bbox_targets = anchors.new_zeros( (num_anchors, self.box_coder.code_size)) if len(gt_boxes) > 0 and anchors.shape[0] > 0: fg_gt_boxes = gt_boxes[anchor_to_gt_argmax[fg_inds], :] fg_anchors = anchors[fg_inds, :] bbox_targets[fg_inds, :] = self.box_coder.encode_torch( fg_gt_boxes, fg_anchors) reg_weights = anchors.new_zeros((num_anchors, )) if self.norm_by_num_examples: num_examples = (labels >= 0).sum() num_examples = num_examples if num_examples > 1.0 else 1.0 reg_weights[labels > 0] = 1.0 / num_examples else: reg_weights[labels > 0] = 1.0 ret_dict = { 'box_cls_labels': labels, 'box_reg_targets': bbox_targets, 'reg_weights': reg_weights, } return ret_dict
def _create_data(self, height=3, width=3, channels=3, device="cpu"): tensor = torch.randint(0, 256, (channels, height, width), dtype=torch.uint8, device=device) pil_img = Image.fromarray(tensor.permute(1, 2, 0).contiguous().cpu().numpy()) return tensor, pil_img
# rand/rand_like, randint 随机初始化 a = torch.rand(3, 3) # 0-1均匀分布 print(a) # tensor([[0.0090, 0.2389, 0.1770], # [0.8717, 0.2723, 0.6541], # [0.0187, 0.8649, 0.3536]]) b = torch.rand_like(a) print(b) # tensor([[0.7082, 0.1881, 0.5463], # [0.4117, 0.0362, 0.3035], # [0.4009, 0.4509, 0.7530]]) c = torch.randint(1, 4, [3, 3]) # low <= rand < high print(c) # tensor([[1, 1, 1], # [2, 2, 3], # [3, 1, 3]]) d = torch.randn(3,3) # 正态分布 N(0,1) print(d) # tensor([[-0.4998, -2.2904, 0.2071], # [ 0.5809, -1.7752, 0.4195], # [ 0.6013, -0.3266, -0.0888]]) e = torch.normal(mean=torch.full([10], 0), std=torch.arange(1, 0, -0.1)) print(e) # tensor([-0.3113, 0.5304, -0.4583, 0.7140, -0.1470, -0.1260, 0.5031, -0.4727, -0.0888, 0.0833])
def NewZerosModuleFloat2D_basic(module, tu: TestUtils): module.forward(torch.randint(10, (2, 3, 4)))
# Print the current values of the losses. if iteration % 10 == 0: avg_transmittance_float = float(avg_transmittance.mean().item()) pbar.set_description( f"Iteration {iteration:05d}:" + f" clip_loss = {float(clip_loss.item()):1.2f}" + f" diffusion_loss = {float(diffusion_loss.item()):1.5f}" + f" avg transmittance = {avg_transmittance_float:1.2f}" + f" tau = {str(tau)}") # Visualize the renders every 100 iterations. if visualize_images: wandb_image = lambda x: wandb.Image(clamp_and_detach(x)) # Visualize only a single randomly selected element of the batch. im_show_idx = int(torch.randint(low=0, high=args.batch_size, size=(1,))) rendering = rendered_images[im_show_idx] silhouette = rendered_silhouettes[im_show_idx] metrics["render/rendered"] = wandb.Image(clamp_and_detach(rendering)) if args.clip_lam > 0: aug_image = clip_aug_images[im_show_idx].movedim(0, 2) metrics["render/augmented"] = wandb_image(aug_image) render_white_bg = rendering * silhouette + 1 - silhouette metrics["render/rendered_white_bg"] = wandb_image(render_white_bg) metrics["render/rendered_silhouettes"] = wandb_image( silhouette.squeeze(-1)) diffused = torch.stack(
def OnesLikeModule_int(module, tu: TestUtils): module.forward(torch.randint(10, (3, 5)))
x = self.avgpool(x) # ([N, 2048, 1, 1]) x = torch.flatten(x,1) # (N,2048) out = self.dp(self.relu(self.fc1_bn(self.fc1(x)))) return out def __make_layer(self,input_channel,hidden_channel,num_blocks): layers = [] for i in range(num_blocks): layers.append(BasicResBlock(input_channel,hidden_channel)) return nn.Sequential(*layers) def create_embedding_layer(self, weight_matrix,input_size, embedding_dims, trainable = False): if weight_matrix is not None: with open(weight_matrix, 'rb') as file: word2vec = pickle.load(file) else: word2vec = np.random.randn(input_size,embedding_dims) word2vec = torch.from_numpy(word2vec) num_embeddings, embedding_dims = word2vec.size() emb_layer = torch.nn.Embedding(num_embeddings, embedding_dims) emb_layer.load_state_dict({'weight': word2vec}) if trainable is False: emb_layer.weight.require_grad = False return emb_layer if __name__ == '__main__': x = torch.randint(0,6411,[16,32]) x = torch.LongTensor(x) print(x.size()) cnn = textCNN(word2vec = None) out = cnn(x) print(out.size())
def summary(Net): clf = Net(1, 10, in_ch=1, debug=True).to(device) data = torch.randn(32, 1, 32, 32).to(device) labels = torch.randint(0, 10, (32, )).to(device) clf(data, labels) print('Nb parameters: {}'.format(nb_parameters(clf)))
def __init__(self, total_count, length): self.data = torch.randint(0, 2, (total_count, length, 1)).type( torch.FloatTensor) self.labels = torch.unsqueeze( torch.FloatTensor([obj_func(xs.flatten()) for xs in self.data]), -1)