def cov_mat(self, x1, x2=None, white_noise_var=None, add_likelihood_var=False): # white_noise_var needs to be passed explicitly x1_ = to_torch(x1) x2_ = to_torch(x2) self.model.eval() with torch.no_grad(): x1_ = self.model.latent_func(x1_) if x2_ is None or torch.equal(x1_, x2_): cov = self.model.kernel_covar_module( x1_).evaluate().cpu().numpy() else: x2_ = self.model.latent_func(x2_) cov = self.model.kernel_covar_module( x1_, x2_).evaluate().cpu().numpy() if white_noise_var is not None: cov += np.diag(white_noise_var) # for training data, add likelihood variance if add_likelihood_var: cov += self.likelihood.log_noise.exp().item() * np.eye( len(cov)) return cov
def extract_feature(self, data_loader): print_freq = 50 self.cnn_model.eval() self.att_model.eval() batch_time = AverageMeter() data_time = AverageMeter() end = time.time() allfeatures = 0 allfeatures_raw = 0 for i, (imgs, flows, _, _) in enumerate(data_loader): imgs = to_torch(imgs) flows = to_torch(flows) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) flows = flows.to(device) with torch.no_grad(): if i == 0: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model(out_feat, out_raw) allfeatures = out_feat allfeatures_raw = out_raw preimgs = imgs preflows = flows elif imgs.size(0) < data_loader.batch_size: flaw_batchsize = imgs.size(0) cat_batchsize = data_loader.batch_size - flaw_batchsize imgs = torch.cat((imgs, preimgs[0:cat_batchsize]), 0) flows = torch.cat((flows, preflows[0:cat_batchsize]), 0) out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model(out_feat, out_raw) out_feat = out_feat[0:flaw_batchsize] out_raw = out_feat[0:flaw_batchsize] allfeatures = torch.cat((allfeatures, out_feat), 0) allfeatures_raw = torch.cat((allfeatures_raw, out_raw), 0) else: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model(out_feat, out_raw) allfeatures = torch.cat((allfeatures, out_feat), 0) allfeatures_raw = torch.cat((allfeatures_raw, out_raw), 0) batch_time.update(time.time() - end) end = time.time() if (i + 1) % print_freq == 0: print('Extract Features: [{}/{}]\t' 'Time {:.3f} ({:.3f})\t' 'Data {:.3f} ({:.3f})\t' .format(i + 1, len(data_loader), batch_time.val, batch_time.avg, data_time.val, data_time.avg)) return allfeatures, allfeatures_raw
def _prep_train_data(self, x, y_ind, y): # prepare training data self._train_x = to_torch(x) self._train_y_ind = to_torch(y_ind).long() train_y = to_torch(y) # single mean estimate across all categories self._train_y_mean = train_y.mean() self._train_y = train_y - self._train_y_mean
def extract_feature(self, data_loader): print_freq = 50 self.cnn_model.eval() self.att_model.eval() batch_time = AverageMeter() data_time = AverageMeter() end = time.time() allfeatures = 0 allfeatures_raw = 0 for i, (imgs, flows, _, _) in enumerate(data_loader): data_time.update(time.time() - end) imgs = to_torch(imgs).to(self.device) flows = to_torch(flows).to(self.device) with torch.no_grad(): if i == 0: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) allfeatures = [out_feat] allfeatures_raw = [out_raw] preimgs = imgs preflows = flows elif imgs.size(0) < data_loader.batch_size: flaw_batchsize = imgs.size(0) cat_batchsize = data_loader.batch_size - flaw_batchsize imgs = torch.cat((imgs, preimgs[0:cat_batchsize]), 0) flows = torch.cat((flows, preflows[0:cat_batchsize]), 0) out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat = out_feat[0:flaw_batchsize] out_raw = out_raw[0:flaw_batchsize] allfeatures.append(out_feat) allfeatures_raw.append(out_raw) else: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) allfeatures.append(out_feat) allfeatures_raw.append(out_raw) batch_time.update(time.time() - end) end = time.time() if (i + 1) % print_freq == 0: print('Extract Features: [{}/{}]\t' 'Time {:.3f} ({:.3f})\t' 'Data {:.3f} ({:.3f})\t' .format(i + 1, len(data_loader), batch_time.val, batch_time.avg, data_time.val, data_time.avg)) allfeatures = torch.cat(allfeatures, 0) allfeatures_raw = torch.cat(allfeatures_raw, 0) return allfeatures, allfeatures_raw
def _fit_net(self, generator, n_steps): self.net.train() # train mode for i, (X_batch, y_batch) in enumerate(islice(generator, n_steps)): X_batch = to_torch(X_batch, cuda=self.cuda_flag) y_batch = to_torch(y_batch, cuda=self.cuda_flag) self.net_optimizer.zero_grad() # zero-out the gradients because they accumulate by default y_pred = self.net.forward(X_batch) loss = self.net_criterion(y_pred, y_batch) self.net_loss.append(loss.item()) loss.backward() # compute gradients self.net_optimizer.step() # update params return self
def set_train_data(self, x, y, var=None): self._train_x = to_torch(x) self._train_y = to_torch(y) self._train_y_mean = self._train_y.mean() self._zero_mean_train_y = self._train_y - self._train_y_mean if var is not None: self._train_var = to_torch(var) if self.model is not None: self.model.set_train_data(inputs=self._train_x, targets=self._zero_mean_train_y, strict=False)
def group_sample(sample, sample_spec, phase="train"): """ Creates the Torch tensors for a sample """ inputs = sample_spec.get_inputs() labels = sample_spec.get_labels() masks = sample_spec.get_masks() input_vars = [utils.to_torch(sample[k], block=True) for k in inputs] label_vars = [utils.to_torch(sample[k], block=False) for k in labels] mask_vars = [utils.to_torch(sample[k], block=False) for k in masks] return input_vars, label_vars, mask_vars
def accuracy(output, target, topk=(1, )): output, target = to_torch(output), to_torch(target) maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) ret = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0) ret.append(correct_k.mul_(1. / batch_size)) return ret
def accuracy(output, target, topk=(1,)): output, target = to_torch(output), to_torch(target) maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) ret = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0) ret.append(correct_k.mul_(1. / batch_size)) return ret
def __init__(self, vocab_size, word_embedding_size, hidden_size, bidirectional=False, input_dropout_p=0, dropout_p=0, n_layers=1, rnn_type='lstm', variable_lengths=True, pretrain=False): super(RNNEncoder, self).__init__() self.variable_lengths = variable_lengths if pretrain is True: embedding_mat = np.load('./data/word_embedding/embed_matrix.npy') self.embedding = nn.Embedding.from_pretrained( to_torch(embedding_mat).cuda(), freeze=False) else: self.embedding = nn.Embedding(vocab_size, word_embedding_size) self.input_dropout = nn.Dropout(input_dropout_p) self.rnn_type = rnn_type self.rnn = getattr(nn, rnn_type.upper())(word_embedding_size, hidden_size, n_layers, batch_first=True, bidirectional=bidirectional, dropout=dropout_p) self.num_dirs = 2 if bidirectional else 1
def extract_cnn_feature(model, inputs, norm_test=True): model.eval() inputs = to_torch(inputs) inputs = Variable(inputs) outputs, _, _ = model(inputs, Norm_test=norm_test) outputs = outputs.data.cpu() return outputs
def extract_feature(self, data_loader): # 2 # print_freq = 50 self.cnn_model.eval() qf = [] # qf_raw = [] for i, inputs in enumerate(data_loader): imgs, _, _ = inputs b, n, s, c, h, w = imgs.size() imgs = imgs.view(b * n, s, c, h, w) imgs = to_torch(imgs) # torch.Size([8, 8, 3, 256, 128]) # flows = to_torch(flows) # torch.Size([8, 8, 3, 256, 128]) device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) # flows = flows.to(device) with torch.no_grad(): out_feat, out_raw = self.cnn_model(imgs) allfeatures = out_feat.view(n, -1) # torch.Size([8, 128]) # allfeatures_raw = out_raw.view(n, -1) # torch.Size([8, 128]) allfeatures = torch.mean(allfeatures, 0).data.cpu() # 汇总一个序列特征,取平均 # allfeatures_raw = torch.mean(allfeatures_raw, 0).data qf.append(allfeatures) # qf_raw.append(allfeatures_raw) qf = torch.stack(qf) # qf_raw = torch.stack(allfeatures_raw) print( "Extracted features for query/gallery set, obtained {}-by-{} matrix" .format(qf.size(0), qf.size(1))) return qf
def _reset(self): self._env = self._get_env( ) # Create new env every episode. This is a way to prevent checkpointing from having to save env state if self.torch_np_conversion: self._state = to_torch(self._env.reset()) else: self._state = self._env.reset() self._step_i = 0
def compute_test(best_model): model = best_model train_predict, hx = model(to_torch(train_x)) train_predict = train_predict.detach().numpy() val_predict, hx = model(to_torch(validate_x), hx) test_predict, _ = model(to_torch(test_x), hx) test_predict = test_predict.detach().numpy() # invert predictions test_predict_r = scaler.inverse_transform(test_predict[:, 0, :]) test_y_r = scaler.inverse_transform(test_y[:, 0, :]) # calculate error test_rmse = math.sqrt( mean_squared_error(test_y_r[:, 0], test_predict_r[:, 0])) test_mape = (abs( (test_predict_r[:, 0] - test_y_r[:, 0]) / test_y_r[:, 0])).mean() test_mae = mean_absolute_error(test_predict_r[:, 0], test_y_r[:, 0]) return test_rmse, test_mape, test_mae
def predict(self, x, y_ind, return_var=False, return_ent=False): # in absence of any training data if self.model is None: if return_ent: return np.full(len(x), 0.0) elif return_var: # assuming rbf kernel with scale = 1 return np.full(len(x), 0.0), np.full(len(x), 1.0) else: raise NotImplementedError( 'Predictive distribution can not be estimated in absence of training data' ) self.model.eval() self.likelihood.eval() ind_ = to_torch(y_ind).long() # TODO: for fast variance computation, add all the relevant flags # fast_pred_var uses LOVE with torch.no_grad(): x_ = to_torch(x) if len(self._train_x) > 10: with gpytorch.fast_pred_var(): pred_grv = self.likelihood(self.model(x_, ind_)) else: pred_grv = self.likelihood(self.model(x_, ind_)) if return_ent: return entropy_from_cov( pred_grv.covariance_matrix.cpu().numpy()) # single mean mu = pred_grv.mean + self._train_y_mean # category-wise mean # mu = pred_grv.mean + torch.gather(self._train_y_mean, 0, ind_) mu = mu.cpu().numpy() if return_var: var = pred_grv.variance.cpu().numpy() return mu, var return mu
def step(self, action): state, reward, done, info = self.env.step(action) with torch.no_grad(): # Just to be sure... t_state = state if self.mode == "torch" else to_torch(state) done_condition = bool(self.env.unwrapped.is_done( t_state).item()) # done do to termination conditions done_timelimit = done if done else False done = done_condition or done_timelimit return state, reward, done, info
def _predict_proba(self, X): y_proba = [] self.net.eval() # evaluation mode for X_batch in OneEpoch(X, batch_size=self.batch_size): X_batch = X_batch.astype(np.float32) with torch.no_grad(): X_batch = to_torch(X_batch, cuda=self.cuda_flag) proba_batch = F.softmax(self.net.forward(X_batch), dim=1).cpu().data.numpy() y_proba.extend(proba_batch) y_proba = np.array(y_proba) return y_proba
def _fit_combined(self, generator, recovery_generator, n_steps): self.net.train() # train mode self.adv_net.train() # train mode for i, (X_batch, y_batch, z_batch) in enumerate(islice(generator, n_steps)): X_batch = to_torch(X_batch, cuda=self.cuda_flag) y_batch = to_torch(y_batch, cuda=self.cuda_flag) z_batch = to_torch(z_batch, cuda=self.cuda_flag) self.net_optimizer.zero_grad() # zero-out the gradients because they accumulate by default y_pred = self.net.forward(X_batch) z_pred = self.adv_net.forward(y_pred) net_loss = self.net_criterion(y_pred, y_batch) adv_loss = self.adv_criterion(z_pred, z_batch) loss = net_loss - (self.trade_off * adv_loss) self.adv_loss.append(adv_loss.item()) self.net_loss.append(net_loss.item()) self.comb_loss.append(loss.item()) loss.backward() # compute gradients self.net_optimizer.step() # update params # Adversarial recovery self._fit_recovery(recovery_generator, self.n_recovery_steps) return self
def step(self, action, video_file_suffix=None): """ Performs a single step in the environment Args: action (numpy[d_action]) video_file_suffix (string, optional, default=None): Suffix added to the end of the video file name Returns: (s, s', done) transition old_state (torch Tensor[1, d_state]) next_states (torch Tensor[1, d_state]) dones (boolean[1]): indicates if episode has terminated (either do to termination condition (is_done) or time limit) """ if self._state is None: self._reset() if self._record: video_file_full_path = self._video_file_base.format( video_file_suffix) next_state, _, done, info = self.env.step( action, filename=video_file_full_path, record_episode=self._record_next_episode) else: next_state, _, done, info = self.env.step(action) if self.torch_np_conversion: next_state = to_torch(next_state) old_state = self._state.detach() self._state = next_state.detach( ) # For more safety (not required, in principle) self._step_i += 1 if self._render: self.env.render() # Note: at the end of the episode next_state != self.state. The former is the part of the # transition while the latter is the current state of the environment (after reset) if done: self.env.close() self._state = None if self._record: self._run.add_artifact( video_file_full_path ) # save video to sacred DB # TODO WJ: This is not nice. Why EnvLoop should know about _run? self._record_next_episode = self._record_in_queue # if there's a pending recording in the queue, record it on the next episode self._record_in_queue = False return old_state, next_state, done
def predict(self, x, return_cov=False, return_std=False): # returns posterior distribution conditioned on training data # call set_train_data method to set a different training data self.model.eval() self.likelihood.eval() x_ = to_torch(x) with torch.no_grad(): pred = self.likelihood(self.model(x_)) pred_mean = (pred.mean() + self._train_y_mean).cpu().numpy() if return_std: return pred_mean, pred.covar().diag().cpu().numpy() elif return_cov: return pred_mean, pred.covar().evaluate().cpu().numpy() return pred_mean
def pred(self, tensor, z=None): X = tensor[:, self.predictors].numpy() if self.interaction: X = self.poly.fit_transform(X) if self.regime: if z is None: X_extra = X.reshape(X.shape[0], 1, -1).repeat(len(self.predictors), 1).reshape(X.shape[0], -1) X = np.hstack([X, X_extra]) else: X = self.add_regime_ind(X, z) y_pred = to_torch(self.model.predict(X)) return y_pred
def run(agent_): env.reset() locs = [env.get_agent_loc()] cumulative_reward, step = 0, 0 while step < max_steps: # get current state s_t = to_torch(env.get_agent_loc().reshape(1, -1)) # take an action a_t, prob_a_t = agent_.choose_action(s_t) # get a reward, make env transition r_t = env.step(a_t) # updates cumulative_reward += r_t * gamma**step step += 1 locs.append(env.get_agent_loc()) # termination condition if env.is_terminal(): break return cumulative_reward, step, locs
def extract_cnn_feature(model, inputs, pool_feature=False, org_feature=False): model.eval() with torch.no_grad(): inputs = to_torch(inputs) inputs = Variable(inputs).cuda() if pool_feature is False: outputs = model(inputs, rot=False, org_feature=org_feature) return outputs else: # Register forward hook for each module outputs = {} def func(m, i, o): outputs['pool_feature'] = o.data.view(n, -1) hook = model.module._modules.get('features').register_forward_hook(func) model(inputs) hook.remove() # print(outputs['pool_feature'].shape) return outputs['pool_feature']
def extract_cnn_feature(model, inputs, modules=None): model.eval() inputs = to_torch(inputs) inputs = Variable(inputs, volatile=True).cuda() if modules is None: outputs = model(inputs) outputs = outputs.data.cpu() return outputs # Register forward hook for each module outputs = OrderedDict() handles = [] for m in modules: outputs[id(m)] = None def func(m, i, o): outputs[id(m)] = o.data.cpu() handles.append(m.register_forward_hook(func)) model(inputs) for h in handles: h.remove() return list(outputs.values())
def extract_feature(self, data_loader): # 2 # print_freq = 50 self.cnn_model.eval() qf = [] # qf_raw = [] for i, inputs in enumerate(data_loader): feature = [] imgs, _, _ = inputs b, n, s, c, h, w = imgs.size() # torch.Size([9, 16, 5, 256, 128]) imgs = imgs.view(b * n, s, c, h, w) imgs = to_torch(imgs) # torch.Size([9, 16, 5, 256, 128]) imgseq_len = imgs.size(0) device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") # flows = flows.to(device) with torch.no_grad(): for i in range(imgseq_len): img = imgs[i].to(device) # torch.Size([16, 5, 256, 128]) out_feat, _, _, _ = self.cnn_model(img, img) # [1, 128] out_feat = out_feat.squeeze().data.cpu() feature.extend(out_feat) features = np.array(feature) features = features.reshape(imgseq_len, -1) features = np.average(features, 0) # allfeatures = features.tolist() allfeatures = torch.from_numpy(features).unsqueeze(0) qf.append(allfeatures) qf = torch.stack(qf) # qf_raw = torch.stack(allfeatures_raw) print( "Extracted features for query/gallery set, obtained {}-by-{} matrix" .format(qf.size(0), qf.size(1))) return qf
def user_mem_init(u_id, device, feature_mem, loading_model, alpha): """ Initialize user memory cube with personalized bias term and attention values :param u_id: User ID :param device: Device choice :param feature_mem: Feature-specific memory component :param loading_model: Loaded model :param alpha: Hyper-parameter :return: Personalized bias term and attention values """ # Path to raw processed data (in Pickle files) path = 'data_prep/processed_data/raw/' # Load the Pickle files u_x1_data = pickle.load( open('{}sample_{}_x1.p'.format(path, str(u_id)), 'rb')) # Convert the user data into PyTorch tensor u_x1 = to_torch([u_x1_data]).to(device) # Get user profile matrix pu = loading_model(u_x1) # Retrieve the personalized bias term and the attention values personalized_bias_term, att_values = feature_mem.read_head(pu, alpha) # Delete variables to save storage del u_x1_data, u_x1, pu return personalized_bias_term, att_values
def evaluate(self, query_loader, gallery_loader, queryinfo, galleryinfo): self.cnn_model.eval() self.att_model.eval() self.classifier_model.eval() querypid = queryinfo.pid # <class 'list'>: [74, 20, 90, 151, 1, 69, 84, 149, 5, 111, -1,..., 71, 139, 36] querycamid = queryinfo.camid # [0, 0, 0 ,...., 0] querytranum = queryinfo.tranum # <class 'list'>: [35, 21, 24, 26, 28, 38, 30, 32, 1, ..., 20, 31, 25, 29] gallerypid = galleryinfo.pid # # <class 'list'>: [74, 20, 90, 151, 1, 69, 84, 149, 5, 111, -1,..., 71, 139, 36] gallerycamid = galleryinfo.camid # [1, 1, 1 ,...., 1] gallerytranum = galleryinfo.tranum # <class 'list'>: [19, 11, 23, 25, 20, 12,..2, 27, 17] query_resfeatures, query_resraw = self.extract_feature(query_loader) # torch.Size([2787, 8, 128]) gallery_resfeatures, gallery_resraw = self.extract_feature(gallery_loader) # torch.Size([2006, 8, 128]) querylen = len(querypid) # 100 gallerylen = len(gallerypid) # 100 # online gallery extraction single_distmat = np.zeros((querylen, gallerylen)) # <class 'tuple'>: (100, 100) q_start = 0 pooled_query = [] with torch.no_grad(): for qind, qnum in enumerate(querytranum): query_feat_tmp = query_resfeatures[q_start:q_start+qnum, :, :] # torch.Size([35, 8, 128]) query_featraw_tmp = query_resraw[q_start:q_start+qnum, :, :] # torch.Size([35, 8, 2048]) pooled_query_tmp, hidden_query_tmp = self.att_model.selfpooling_model(query_feat_tmp, query_featraw_tmp) # torch.Size([35, 128]) pooled_query.append(pooled_query_tmp) q_start += qnum pooled_query = torch.cat(pooled_query, 0) # torch.Size([2787, 128]) g_start = 0 pooled_gallery = [] with torch.no_grad(): for gind, gnum in enumerate(gallerytranum): gallery_feat_tmp = gallery_resfeatures[g_start:g_start+gnum, :, :] # torch.Size([19, 8, 128]) gallery_featraw_tmp = gallery_resraw[g_start:g_start+gnum, :, :] # torch.Size([19, 8, 2048]) pooled_gallery_tmp, hidden_gallery_tmp = self.att_model.selfpooling_model(gallery_feat_tmp, gallery_featraw_tmp) # torch.Size([19, 128]) pooled_gallery.append(pooled_gallery_tmp) g_start += gnum pooled_gallery = torch.cat(pooled_gallery, 0) # torch.Size([2006, 128]) # pooled_query, hidden_query = self.att_model.selfpooling_model_1(query_resfeatures, query_resraw) # pooled_gallery, hidden_gallery = self.att_model.selfpooling_model_2(gallery_resfeatures, gallery_resraw) pooled_query_2 = self.getcrosspool(query_resfeatures, query_resraw, pooled_gallery, querytranum) pooled_query_2 = to_numpy(pooled_query_2) torch.cuda.empty_cache() pooled_gallery_2 = self.getcrosspool(gallery_resfeatures, gallery_resraw, pooled_query, gallerytranum) pooled_gallery_2 = to_numpy(pooled_gallery_2) torch.cuda.empty_cache() pooled_query_2 = to_torch(pooled_query_2).to(self.device) pooled_gallery_2 = to_torch(pooled_gallery_2).to(self.device) # q_start = 0 # pooled_query_2 = [] # with torch.no_grad(): # for qind, qnum in enumerate(querytranum): # query_feat_tmp = query_resfeatures[q_start:q_start+qnum, :, :] # query_featraw_tmp = query_resraw[q_start:q_start+qnum, :, :] # pooled_query_2_tmp = self.att_model.crosspooling_model(query_feat_tmp, query_featraw_tmp, pooled_gallery) # pooled_query_2.append(pooled_query_2_tmp) # q_start += qnum # torch.cuda.empty_cache() # # torch.cuda.empty_cache() # pooled_query_2 = torch.cat(pooled_query_2, 1) # torch.cuda.empty_cache() # g_start = 0 # pooled_gallery_2 = [] # with torch.no_grad(): # for gind, gnum in enumerate(gallerytranum): # gallery_feat_tmp = gallery_resfeatures[g_start:g_start+gnum, :, :] # torch.Size([19, 8, 128]) # gallery_featraw_tmp = gallery_resraw[g_start:g_start+gnum, :, :] # torch.Size([19, 8, 2048]) # pooled_gallery_2_tmp = self.att_model.crosspooling_model(gallery_feat_tmp, gallery_featraw_tmp, pooled_query) # # torch.Size([2787, 19, 128]) # pooled_gallery_2.append(pooled_gallery_2_tmp) # g_start += gnum # torch.cuda.empty_cache() # # torch.cuda.empty_cache() # pooled_gallery_2 = torch.cat(pooled_gallery_2, 1) pooled_query_2 = pooled_query_2.permute(1, 0, 2) # torch.Size([2787, 2006, 128]) pooled_query, pooled_gallery = pooled_query.unsqueeze(1), pooled_gallery.unsqueeze(0) # torch.Size([2787, 1, 128]) torch.Size([1, 2006, 128]) with torch.no_grad(): encode_scores = self.classifier_model(pooled_query, pooled_gallery_2, pooled_query_2, pooled_gallery) encode_scores = to_torch(encode_scores).to(self.device) encode_size = encode_scores.size() # torch.Size([2787, 2006, 2]) encodemat = encode_scores.view(-1, 2) # torch.Size([5590722, 2]) encodemat = F.softmax(encodemat) encodemat = encodemat.view(encode_size[0], encode_size[1], 2) # torch.Size([2787, 2006, 2]) single_distmat_all = encodemat[:, :, 0] # torch.Size([2787, 2006]) single_distmat_all = single_distmat_all.data.cpu().numpy() q_start, g_start = 0, 0 for qind, qnum in enumerate(querytranum): for gind, gnum in enumerate(gallerytranum): distmat_qg = single_distmat_all[q_start:q_start+qnum, g_start:g_start+gnum] # percile = np.percentile(distmat_qg, 20) percile = np.percentile(distmat_qg, 20) if distmat_qg[distmat_qg <= percile] is not None: distmean = np.mean(distmat_qg[distmat_qg <= percile]) else: distmean = np.mean(distmat_qg) single_distmat[qind, gind] = distmean g_start = g_start + gnum g_start = 0 q_start = q_start + qnum return evaluate_seq(single_distmat, querypid, querycamid, gallerypid, gallerycamid)
# prealloc log_return = [] log_steps = [] for i in range(n_trials): env.reset() cumulative_reward = 0 step = 0 while step < max_steps: if step == 0: h_prev = get_init_state() # get current state to predict action value s_t = to_torch(env.get_agent_loc().reshape(1, -1)) out_t, h_t = rnn(s_t.view(1, 1, -1), h_prev) q_t = readout(out_t) # epsilon greedy action selection if np.random.uniform() > epsilon: a_t = torch.argmax(q_t) else: a_t = np.random.randint(n_actions) # transition and get reward r_t = env.step(a_t) # get next states info s_next = to_torch(env.get_agent_loc().reshape(1, -1)) out_next, _ = rnn(s_next.view(1, 1, -1), h_t) q_next = readout(out_next)
def evaluate(self, query_loader, gallery_loader, queryinfo, galleryinfo): self.cnn_model.eval() self.att_model.eval() self.classifier_model.eval() querypid = queryinfo.pid querycamid = queryinfo.camid querytranum = queryinfo.tranum gallerypid = galleryinfo.pid gallerycamid = galleryinfo.camid gallerytranum = galleryinfo.tranum pooled_probe, hidden_probe = self.extract_feature(query_loader) querylen = len(querypid) gallerylen = len(gallerypid) # online gallery extraction single_distmat = np.zeros((querylen, gallerylen)) gallery_resize = 0 gallery_popindex = 0 gallery_popsize = gallerytranum[gallery_popindex] gallery_resfeatures = 0 gallery_resraw = 0 gallery_empty = True preimgs = 0 preflows = 0 # time gallery_time = AverageMeter() end = time.time() for i, (imgs, flows, _, _) in enumerate(gallery_loader): imgs = to_torch(imgs) flows = to_torch(flows) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) flows = flows.to(device) with torch.no_grad(): seqnum = imgs.size(0) if i == 0: preimgs = imgs preflows = flows if gallery_empty: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) gallery_resfeatures = out_feat gallery_resraw = out_raw gallery_empty = False elif imgs.size(0) < gallery_loader.batch_size: flaw_batchsize = imgs.size(0) cat_batchsize = gallery_loader.batch_size - flaw_batchsize imgs = torch.cat((imgs, preimgs[0:cat_batchsize]), 0) flows = torch.cat((flows, preflows[0:cat_batchsize]), 0) out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat = out_feat[0:flaw_batchsize] out_raw = out_raw[0:flaw_batchsize] gallery_resfeatures = torch.cat((gallery_resfeatures, out_feat), 0) gallery_resraw = torch.cat((gallery_resraw, out_raw), 0) else: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) gallery_resfeatures = torch.cat((gallery_resfeatures, out_feat), 0) gallery_resraw = torch.cat((gallery_resraw, out_raw), 0) gallery_resize = gallery_resize + seqnum while gallery_popsize <= gallery_resize: if (gallery_popindex + 1) % 50 == 0: print('gallery--{:04d}'.format(gallery_popindex)) gallery_popfeatures = gallery_resfeatures[0:gallery_popsize, :] gallery_popraw = gallery_resraw[0:gallery_popsize, :] if gallery_popsize < gallery_resize: gallery_resfeatures = gallery_resfeatures[gallery_popsize:gallery_resize, :] gallery_resraw = gallery_resraw[gallery_popsize:gallery_resize, :] else: gallery_resfeatures = 0 gallery_resraw = 0 gallery_empty = True gallery_resize = gallery_resize - gallery_popsize pooled_gallery, pooled_raw = self.att_model.selfpooling_model(gallery_popfeatures, gallery_popraw) probesize = pooled_probe.size() gallerysize = pooled_gallery.size() probe_batch = probesize[0] gallery_batch = gallerysize[0] gallery_num = gallerysize[1] pooled_gallery.unsqueeze(0) pooled_gallery = pooled_gallery.expand(probe_batch, gallery_batch, gallery_num) encode_scores = self.classifier_model(pooled_probe, pooled_gallery) encode_size = encode_scores.size() encodemat = encode_scores.view(-1, 2) encodemat = F.softmax(encodemat) encodemat = encodemat.view(encode_size[0], encode_size[1], 2) distmat_qall_g = encodemat[:, :, 0] q_start = 0 for qind, qnum in enumerate(querytranum): distmat_qg = distmat_qall_g[q_start:q_start + qnum, :] distmat_qg = distmat_qg.data.cpu().numpy() percile = np.percentile(distmat_qg, 20) if distmat_qg[distmat_qg <= percile] is not None: distmean = np.mean(distmat_qg[distmat_qg <= percile]) else: distmean = np.mean(distmat_qg) single_distmat[qind, gallery_popindex] = distmean q_start = q_start + qnum gallery_popindex = gallery_popindex + 1 if gallery_popindex < gallerylen: gallery_popsize = gallerytranum[gallery_popindex] gallery_time.update(time.time() - end) end = time.time() return evaluate_seq(single_distmat, querypid, querycamid, gallerypid, gallerycamid)
def make_variables(inputs): expanded = [np.expand_dims(arr, axis=0) for (k, arr) in inputs.items()] return [utils.to_torch(arr) for arr in expanded]
def evaluate(self, query_loader, gallery_loader, queryinfo, galleryinfo): self.cnn_model.eval() self.att_model.eval() self.classifier_model.eval() querypid = queryinfo.pid querycamid = queryinfo.camid querytranum = queryinfo.tranum gallerypid = galleryinfo.pid gallerycamid = galleryinfo.camid gallerytranum = galleryinfo.tranum pooled_probe, hidden_probe = self.extract_feature(query_loader) querylen = len(querypid) gallerylen = len(gallerypid) # online gallery extraction single_distmat = np.zeros((querylen, gallerylen)) gallery_resize = 0 gallery_popindex = 0 gallery_popsize = gallerytranum[gallery_popindex] gallery_resfeatures = 0 gallery_resraw = 0 gallery_empty = True preimgs = 0 preflows = 0 # time gallery_time = AverageMeter() end = time.time() for i, (imgs, flows, _, _) in enumerate(gallery_loader): imgs = to_torch(imgs) flows = to_torch(flows) device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) flows = flows.to(device) with torch.no_grad(): seqnum = imgs.size(0) if i == 0: preimgs = imgs preflows = flows if gallery_empty: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) gallery_resfeatures = out_feat gallery_resraw = out_raw gallery_empty = False elif imgs.size(0) < gallery_loader.batch_size: flaw_batchsize = imgs.size(0) cat_batchsize = gallery_loader.batch_size - flaw_batchsize imgs = torch.cat((imgs, preimgs[0:cat_batchsize]), 0) flows = torch.cat((flows, preflows[0:cat_batchsize]), 0) out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat = out_feat[0:flaw_batchsize] out_raw = out_raw[0:flaw_batchsize] gallery_resfeatures = torch.cat( (gallery_resfeatures, out_feat), 0) gallery_resraw = torch.cat((gallery_resraw, out_raw), 0) else: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) gallery_resfeatures = torch.cat( (gallery_resfeatures, out_feat), 0) gallery_resraw = torch.cat((gallery_resraw, out_raw), 0) gallery_resize = gallery_resize + seqnum while gallery_popsize <= gallery_resize: if (gallery_popindex + 1) % 50 == 0: print('gallery--{:04d}'.format(gallery_popindex)) gallery_popfeatures = gallery_resfeatures[0:gallery_popsize, :] gallery_popraw = gallery_resraw[0:gallery_popsize, :] if gallery_popsize < gallery_resize: gallery_resfeatures = gallery_resfeatures[ gallery_popsize:gallery_resize, :] gallery_resraw = gallery_resraw[ gallery_popsize:gallery_resize, :] else: gallery_resfeatures = 0 gallery_resraw = 0 gallery_empty = True gallery_resize = gallery_resize - gallery_popsize pooled_gallery, pooled_raw = self.att_model.selfpooling_model( gallery_popfeatures, gallery_popraw) probesize = pooled_probe.size() gallerysize = pooled_gallery.size() probe_batch = probesize[0] gallery_batch = gallerysize[0] gallery_num = gallerysize[1] pooled_gallery.unsqueeze(0) pooled_gallery = pooled_gallery.expand(probe_batch, gallery_batch, gallery_num) encode_scores = self.classifier_model(pooled_probe, pooled_gallery) encode_size = encode_scores.size() encodemat = encode_scores.view(-1, 2) encodemat = F.softmax(encodemat) encodemat = encodemat.view(encode_size[0], encode_size[1], 2) distmat_qall_g = encodemat[:, :, 0] q_start = 0 for qind, qnum in enumerate(querytranum): distmat_qg = distmat_qall_g[q_start:q_start + qnum, :] distmat_qg = distmat_qg.data.cpu().numpy() percile = np.percentile(distmat_qg, 20) if distmat_qg[distmat_qg <= percile] is not None: distmean = np.mean(distmat_qg[distmat_qg <= percile]) else: distmean = np.mean(distmat_qg) single_distmat[qind, gallery_popindex] = distmean q_start = q_start + qnum gallery_popindex = gallery_popindex + 1 if gallery_popindex < gallerylen: gallery_popsize = gallerytranum[gallery_popindex] gallery_time.update(time.time() - end) end = time.time() return evaluate_seq(single_distmat, querypid, querycamid, gallerypid, gallerycamid)
def extract_feature(self, data_loader): self.cnn_model.eval() self.siamese_model.eval() qf, q_pids, q_camids = [], [], [] for i, inputs in enumerate(data_loader): imgs, pids, camids = inputs if self.only_eval: b, n, s, c, h, w = imgs.size() # 1, 5, 8, c, h, w imgs = imgs.view(b * n, s, c, h, w).cuda() with torch.no_grad(): if b * n > 8: # 如果序列过长,则分成若干个15个batch_size feat_list = [] # 弄一个临时列表,存放特征 num = int(math.ceil(b * n * 1.0 / 8)) # 有几个32 for y in range(num): clips = imgs[ y * 8:(y + 1) * 8, :, :, :, :].cuda() # 32, 8, c, h, w x_uncorr, feats_corr = self.cnn_model(clips) out_frame = self.siamese_model.self_attention( feats_corr) out_feat = torch.cat( (x_uncorr, out_frame, feats_corr.mean(dim=1)), dim=1) feat_list.append(out_feat) feat_list = torch.cat(feat_list, 0) feat_list = torch.mean(feat_list, dim=0) qf.append(feat_list.unsqueeze(0)) q_pids.extend(pids) q_camids.extend(camids) else: x_uncorr, feats_corr = self.cnn_model(imgs) out_frame = self.siamese_model.self_attention( feats_corr) out_feat = torch.cat( (x_uncorr, out_frame, feats_corr.mean(dim=1)), dim=1) out_feat = out_feat.view(n, -1) out_feat = torch.mean(out_feat, dim=0) qf.append(out_feat.unsqueeze(0)) q_pids.extend(pids) q_camids.extend(camids) torch.cuda.empty_cache() else: b, s, c, h, w = imgs.size() imgs = imgs.view(b, s, c, h, w) imgs = to_torch(imgs) device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) with torch.no_grad(): x_uncorr, feats_corr = self.cnn_model(imgs) out_frame = self.siamese_model.self_attention(feats_corr) out_feat = torch.cat( (x_uncorr, out_frame, feats_corr.mean(dim=1)), dim=1) qf.append(out_feat) q_pids.extend(pids) q_camids.extend(camids) torch.cuda.empty_cache() qf = torch.cat(qf, 0) q_pids = np.asarray(q_pids) q_camids = np.asarray(q_camids) return qf, q_pids, q_camids
def extract_feature(self, data_loader): print_freq = 50 self.cnn_model.eval() self.att_model.eval() batch_time = AverageMeter() data_time = AverageMeter() end = time.time() allfeatures = 0 allfeatures_raw = 0 for i, (imgs, flows, _, _) in enumerate(data_loader): imgs = to_torch(imgs) flows = to_torch(flows) device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") imgs = imgs.to(device) flows = flows.to(device) with torch.no_grad(): if i == 0: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model( out_feat, out_raw) allfeatures = out_feat allfeatures_raw = out_raw preimgs = imgs preflows = flows elif imgs.size(0) < data_loader.batch_size: flaw_batchsize = imgs.size(0) cat_batchsize = data_loader.batch_size - flaw_batchsize imgs = torch.cat((imgs, preimgs[0:cat_batchsize]), 0) flows = torch.cat((flows, preflows[0:cat_batchsize]), 0) out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model( out_feat, out_raw) out_feat = out_feat[0:flaw_batchsize] out_raw = out_feat[0:flaw_batchsize] allfeatures = torch.cat((allfeatures, out_feat), 0) allfeatures_raw = torch.cat((allfeatures_raw, out_raw), 0) else: out_feat, out_raw = self.cnn_model(imgs, flows, self.mode) out_feat, out_raw = self.att_model.selfpooling_model( out_feat, out_raw) allfeatures = torch.cat((allfeatures, out_feat), 0) allfeatures_raw = torch.cat((allfeatures_raw, out_raw), 0) batch_time.update(time.time() - end) end = time.time() if (i + 1) % print_freq == 0: print('Extract Features: [{}/{}]\t' 'Time {:.3f} ({:.3f})\t' 'Data {:.3f} ({:.3f})\t'.format(i + 1, len(data_loader), batch_time.val, batch_time.avg, data_time.val, data_time.avg)) return allfeatures, allfeatures_raw