def seq_data_iter_random(corpus, batch_size, num_steps): # Offset the iterator over the data for uniform starts corpus = corpus[random.randint(0, num_steps):] # Subtract 1 extra since we need to account for label num_examples = ((len(corpus) - 1) // num_steps) example_indices = list(range(0, num_examples * num_steps, num_steps)) random.shuffle(example_indices) def data(pos): # This returns a sequence of the length num_steps starting from pos return corpus[pos:pos + num_steps] # Discard half empty batches num_batches = num_examples // batch_size for i in range(0, batch_size * num_batches, batch_size): # Batch_size indicates the random examples read each time batch_indices = example_indices[i:(i + batch_size)] X = [data(j) for j in batch_indices] Y = [data(j + 1) for j in batch_indices] yield torch.Tensor(X), torch.Tensor(Y)
def seq_data_iter_random(corpus, batch_size, num_steps): """使用随机抽样生成一小批子序列。""" # 从随机偏移量(包括`num_steps - 1`)开始对序列进行分区 corpus = corpus[random.randint(0, num_steps - 1):] # 减去1,因为我们需要考虑标签 num_subseqs = (len(corpus) - 1) // num_steps # 长度为`num_steps`的子序列的起始索引 initial_indices = list(range(0, num_subseqs * num_steps, num_steps)) # 在随机抽样中,迭代过程中两个相邻随机小批量的子序列不一定在原始序列上相邻 random.shuffle(initial_indices) def data(pos): # 返回从`pos`开始的长度为`num_steps`的序列 return corpus[pos:pos + num_steps] num_batches = num_subseqs // batch_size for i in range(0, batch_size * num_batches, batch_size): # 这里,`initial_indices`包含子序列的随机起始索引 initial_indices_per_batch = initial_indices[i:i + batch_size] X = [data(j) for j in initial_indices_per_batch] Y = [data(j + 1) for j in initial_indices_per_batch] yield d2l.tensor(X), d2l.tensor(Y)
def _get_words_dict(self, data, max_words): word_counter = Counter(w.lower_ for d in self.nlp.tokenizer.pipe(( doc for doc in tqdm(data(), desc="Tokenizing data"))) for w in d) dict_w = { w: i for i, (w, _) in tqdm(enumerate( word_counter.most_common(max_words), start=2), desc="building word dict", total=max_words) } dict_w["_padding_"] = 0 dict_w["_unk_word_"] = 1 print("Dictionnary has {} words".format(len(dict_w))) return dict_w
def seq_data_iter_random(corpus, batch_size, num_steps): #@save # Start with a random offset to partition a sequence corpus = corpus[random.randint(0, num_steps):] # Subtract 1 since we need to account for labels num_subseqs = (len(corpus) - 1) // num_steps # The starting indices for subsequences of length `num_steps` initial_indices = list(range(0, num_subseqs * num_steps, num_steps)) # In random sampling, the subsequences from two adjacent random # minibatches during iteration are not necessarily adjacent on the # original sequence random.shuffle(initial_indices) def data(pos): # Return a sequence of length `num_steps` starting from `pos` return corpus[pos: pos + num_steps] num_subseqs_per_example = num_subseqs // batch_size for i in range(0, batch_size * num_subseqs_per_example, batch_size): # Here, `initial_indices` contains randomized starting indices for # subsequences initial_indices_per_batch = initial_indices[i: i + batch_size] X = [data(j) for j in initial_indices_per_batch] Y = [data(j + 1) for j in initial_indices_per_batch] yield d2l.tensor(X), d2l.tensor(Y)
def dataset(num_trials: int, n_time: int, Inp_dim: int, tau: int = 1000, latents: Tuple[Tuple[str]] = (('x',), ('y',), ('theta',)), num_peaks: Optional[Tuple[int]] = None, peak_width_factors: Optional[Union[Tuple[Tuple[float]], float]] = 0.1, sigma_mult: float = 0., sigma_add: float = 0., freeze_epochs: bool = True, train_perc: float = 0.8, return_latents: bool = False): if not freeze_epochs: raise AttributeError("freeze_epochs=False option isn't implemented yet") else: # class LatentVariableReceptiveFieldEncoding(torch.utils.data.Dataset): # def __init__(self): # pass inputs, targets, latent_vals = data(num_trials, n_time, Inp_dim, tau, latents, num_peaks, peak_width_factors, sigma_mult, sigma_add) train_time = int(round(train_perc * n_time)) train_inputs = inputs[:, :train_time] train_inputs = train_inputs.reshape(-1, train_inputs.shape[-1]) val_inputs = inputs[:, train_time:] val_inputs = val_inputs.reshape(-1, val_inputs.shape[-1]) train_targets = targets[:, :train_time] train_targets = train_targets.reshape(-1, train_targets.shape[-1]) val_targets = targets[:, train_time:] val_targets = val_targets.reshape(-1, val_targets.shape[-1]) train_dataset = InpData(train_inputs, train_targets) val_dataset = InpData(val_inputs, val_targets) train_latent_vals = [] val_latent_vals = [] for group in latent_vals: train_latent_vals.append([]) val_latent_vals.append([]) for x in group: train_latent_vals[-1].append(x[:, :train_time].reshape(-1)) val_latent_vals[-1].append(x[:, train_time:].reshape(-1)) if return_latents: return train_dataset, val_dataset, train_latent_vals, val_latent_vals else: return train_dataset, val_dataset
trace.save('resnet18.pt') print('---') print(trace.graph) data = torch.jit.load('resnet18.pt') input = torch.rand(64, 3, 224, 224) s = 0 s2 = 0 for i in range(0, 30): t = time.time() data(input) s = time.time() - t t = time.time() model(input) s2 = time.time() - t print('JIT {}'.format(s)) print('Python: {}'.format(s2)) s = 0 s2 = 0 for i in range(0, 30): t = time.time() data(input)
LOG_FOUT = open(os.path.join(save_dir, 'log_test.txt'), 'w') LOG_FOUT.write(str(save_dir) + '\n') viz_dir = os.path.join(save_dir, 'visualize') train_viz = os.path.join(viz_dir, 'train') test_viz = os.path.join(viz_dir, 'test') if not os.path.exists(viz_dir): os.mkdir(viz_dir) os.mkdir(train_viz) os.mkdir(test_viz) os.environ['CUDA_VISIBLE_DEVICES'] = '3' if not test_model: raise NameError('please set the test_model file to choose the checkpoint!') # read dataset trainset = data(root=data_dir, is_train=True, data_len=100) trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8, drop_last=False) testset = data(root=data_dir, is_train=False, data_len=100) testloader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=8, drop_last=False) # define model net = model.attention_net(topN=PROPOSAL_NUM) log_string('Loading {}'.format(test_model))
def main(): model=Tiny(200) model=init_weight(model) model.to(device) if not os.path.exists('./Tiny_IBP_Trained_Models_lr_0.001_clean_train'): os.makedirs('./Tiny_IBP_Trained_Models_lr_0.001_clean_train') method = 'BCP_model' output_dir = './Tiny_IBP_Trained_Models_lr_0.001_clean_train/' start_epoch = 0 epochs = 500 batch_counter = 0 LR=args.lr optimizer = optim.Adam(model.parameters(), lr=LR) EPS_TRAIN=1/255 EPS_TEST=1/255 eps_sch=generate_epsilon_schedule_ImageNet(EPS_TRAIN) kappa_sch=generate_kappa_schedule_ImageNet() # ***********************************dataset setup *********************************************** train_dataset = data('train', transform=transforms.Compose([transforms.RandomCrop(64, padding=4), transforms.RandomHorizontalFlip(),transforms.ToTensor()])) val_dataset = data('val', transform=transforms.Compose([transforms.ToTensor()])) batch_size = 32 train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True) val_loader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=False) # ************************************start training*********************************************** clean_err_list = [] normal_loss_list = [] reg_loss_list = [] a_txt=open(args.out, 'w') print(args,file=a_txt) a_txt.close() for epoch in range(start_epoch, epochs): print("This is ## Tiny ImageNet ## epoch: ", epoch) robust_loss=0 robust_err=0 if epoch>=args.warm: model,normal_loss,robust_loss,reg_loss=robust_train(train_loader, model, eps_sch, device, kappa_sch, batch_counter, optimizer) else: model, normal_loss, reg_loss = clean_train(train_loader, model, device,opt=optimizer) batch_counter += 1 test_loss, prec1, prec5 = clean_test(val_loader, model, device) if epoch>=args.warm: robust_err,robust_loss=robust_test(val_loader,model,EPS_TEST,device) save_name = os.path.join(output_dir, '{}_epoch_{}_robust_{}.h5'.format(method, epoch,1-robust_err)) save_net(save_name, model) clean_err_list.append(test_loss) normal_loss_list.append(normal_loss) reg_loss_list.append(reg_loss) if epoch==args.drop: for param_group in optimizer.param_groups: param_group["lr"] = LR*0.1 train_loss_txt = open(args.out, 'a') train_loss_txt.write('\nepoch %s ' % str(epoch)) train_loss_txt.write('\nnormal_loss %s ' % str(normal_loss_list[epoch])) train_loss_txt.write('\nreg_loss %s ' % str(reg_loss_list[epoch])) train_loss_txt.write('\nprec5 %s ' % str(prec5 )) train_loss_txt.write('\nprec1 %s ' % str( prec1)) train_loss_txt.write('\nrobust prec1 %s ' % str( 1-robust_err)) train_loss_txt.write('\nrobust loss %s ' % str( robust_loss)) train_loss_txt.write('\n') train_loss_txt.close() print('epoch %s ' % str(epoch)) print('normal_loss %s ' % str(normal_loss_list[epoch])) print('reg_loss %s ' % str(reg_loss_list[epoch])) print('prec5 %s ' % str(prec5 )) print('prec1 %s ' % str( prec1)) print('robust prec1 %s ' % str( 1-robust_err)) print('robust loss %s ' % str( robust_loss))