def __init__(self, model, batch_size, hparam_str, session): """Create the model. Args: model: (BasicModel) The model we are going to train. batch_size: the size of the batches generated in each iteration. hparam_str: the hyper-parameters for the input layer. session: the current tensorflow Session (used for online learning). """ self.hparams = ultra.utils.hparams.HParams( # the setting file for the predefined click models. click_model_json='./example/ClickModel/pbm_0.1_1.0_4_1.0.json', # Scalar for the probability distribution. tau=1, # Set True to feed relevance labels instead of simulated clicks. oracle_mode=False, # Set eta change step for dynamic bias severity in training, 0.0 # means no change. dynamic_bias_eta_change=0.0, # Set how many steps to change eta for dynamic bias severity in # training, 0.0 means no change. dynamic_bias_step_interval=1000, ) print('Create online stochastic simluation feed') print(hparam_str) self.hparams.parse(hparam_str) self.click_model = None with open(self.hparams.click_model_json) as fin: model_desc = json.load(fin) self.click_model = cm.loadModelFromJson(model_desc) self.start_index = 0 self.count = 1 self.rank_list_size = model.rank_list_size self.max_candidate_num = model.max_candidate_num self.feature_size = model.feature_size self.batch_size = batch_size self.model = model self.session = session self.global_batch_count = 0 # Check whether the model needs result interleaving. self.need_interleave = False if hasattr(model.hparams, 'need_interleave'): self.need_interleave = model.hparams.need_interleave print('Online simulation with interleaving: %s' % (str(self.need_interleave))) if self.need_interleave: self.interleaving = TeamDraftInterleaving()
class InterleavingDeterministicOnlineSimulationFeed(BaseInputFeed): """Simulate online learning to rank and click data based on human annotations. This class implements a input layer for online learning to rank experiments by simulating click data based on both the human relevance annotation of each query-document pair and a predefined click model. """ def __init__(self, model, batch_size, hparam_str, session): """Create the model. Args: model: (BasicModel) The model we are going to train. batch_size: the size of the batches generated in each iteration. hparam_str: the hyper-parameters for the input layer. session: the current tensorflow Session (used for online learning). """ self.hparams = ultra.utils.hparams.HParams( # the setting file for the predefined click models. click_model_json='./example/ClickModel/pbm_0.1_1.0_4_1.0.json', # Set True to feed relevance labels instead of simulated clicks. oracle_mode=False, # Set eta change step for dynamic bias severity in training, 0.0 # means no change. dynamic_bias_eta_change=0.0, # Set how many steps to change eta for dynamic bias severity in # training, 0.0 means no change. dynamic_bias_step_interval=1000, ) print('Create online simluation feed') print(hparam_str) self.hparams.parse(hparam_str) self.click_model = None with open(self.hparams.click_model_json) as fin: model_desc = json.load(fin) self.click_model = cm.loadModelFromJson(model_desc) self.start_index = 0 self.count = 1 self.rank_list_size = model.rank_list_size self.max_candidate_num = model.max_candidate_num self.feature_size = model.feature_size self.batch_size = batch_size self.model = model self.session = session self.global_batch_count = 0 self.interleaving = TeamDraftInterleaving() def prepare_true_labels_with_index(self, data_set, index, docid_inputs, letor_features, labels, check_validation=False): i = index # Generate label list. label_list = [ 0 if data_set.initial_list[i][x] < 0 else data_set.labels[i][x] for x in range(self.max_candidate_num) ] # Check if data is valid if check_validation and sum(label_list) == 0: return base = len(letor_features) for x in range(self.max_candidate_num): if data_set.initial_list[i][x] >= 0: letor_features.append( data_set.features[data_set.initial_list[i][x]]) docid_inputs.append( list([ -1 if data_set.initial_list[i][x] < 0 else base + x for x in range(self.max_candidate_num) ])) labels.append(label_list) def simulate_clicks_online(self, input_feed, check_validation=False): """Simulate online environment by reranking documents and collect clicks. Args: input_feed: (dict) The input_feed data. check_validation: (bool) Set True to ignore data with no positive labels. Returns: input_feed: a feed dictionary for the next step info_map: a dictionary contain some basic information about the batch (for debugging). """ # Compute ranking scores with input_feed input_feed[self.model.is_training.name] = False rank_scores = self.session.run([self.model.output], input_feed)[0] # Rerank documents and collect clicks letor_features_length = len(input_feed[self.model.letor_features.name]) for i in range(len(input_feed[self.model.docid_inputs[0].name])): # Get valid doc index valid_idx = self.max_candidate_num - 1 while valid_idx > -1: if input_feed[self.model.docid_inputs[valid_idx]. name][i] < letor_features_length: # a valid doc break valid_idx -= 1 list_len = valid_idx + 1 # Rerank documents via interleaving # print(rank_scores[0],'!', rank_scores[1],'!', rank_scores[2]) old_scores = rank_scores[1][i][:list_len] new_scores = rank_scores[2][i][:list_len] old_rank_list = sorted(range(len(old_scores)), key=lambda k: old_scores[k], reverse=True) new_rank_list = sorted(range(len(new_scores)), key=lambda k: new_scores[k], reverse=True) # print([old_rank_list, new_rank_list]) rerank_list = self.interleaving.interleave( np.asarray([old_rank_list, new_rank_list])) # print (rerank_list, list_len) # rerank_list = sorted(range(len(scores)), key=lambda k: scores[k], reverse=True) new_docid_list = np.zeros(list_len) new_label_list = np.zeros(list_len) for j in range(list_len): new_docid_list[j] = input_feed[self.model.docid_inputs[ rerank_list[j]].name][i] new_label_list[j] = input_feed[self.model.labels[ rerank_list[j]].name][i] # Collect clicks online click_list = None if self.hparams.oracle_mode: click_list = new_label_list[:self.rank_list_size] else: click_list, _, _ = self.click_model.sampleClicksForOneList( new_label_list[:self.rank_list_size]) sample_count = 0 while check_validation and sum( click_list ) == 0 and sample_count < self.MAX_SAMPLE_ROUND_NUM: click_list, _, _ = self.click_model.sampleClicksForOneList( new_label_list[:self.rank_list_size]) sample_count += 1 # update input_feed for j in range(list_len): input_feed[ self.model.docid_inputs[j].name][i] = new_docid_list[j] if j < self.rank_list_size: input_feed[self.model.labels[j].name][i] = click_list[j] else: input_feed[self.model.labels[j].name][i] = 0 # Infer winner in interleaving input_feed[self.model.winners. name][i] = self.interleaving.infer_winner(click_list) return input_feed def get_batch(self, data_set, check_validation=False): """Get a random batch of data, prepare for step. Typically used for training. To feed data in step(..) it must be a list of batch-major vectors, while data here contains single length-major cases. So the main logic of this function is to re-index data cases to be in the proper format for feeding. Args: data_set: (Raw_data) The dataset used to build the input layer. check_validation: (bool) Set True to ignore data with no positive labels. Returns: input_feed: a feed dictionary for the next step info_map: a dictionary contain some basic information about the batch (for debugging). """ if len(data_set.initial_list[0]) < self.rank_list_size: raise ValueError( "Input ranklist length must be no less than the required list size," " %d != %d." % (len(data_set.initial_list[0]), self.rank_list_size)) length = len(data_set.initial_list) docid_inputs, letor_features, labels = [], [], [] rank_list_idxs = [] for _ in range(self.batch_size): i = int(random.random() * length) rank_list_idxs.append(i) self.prepare_true_labels_with_index(data_set, i, docid_inputs, letor_features, labels, check_validation) local_batch_size = len(docid_inputs) letor_features_length = len(letor_features) for i in range(local_batch_size): for j in range(self.max_candidate_num): if docid_inputs[i][j] < 0: docid_inputs[i][j] = letor_features_length batch_docid_inputs = [] batch_labels = [] for length_idx in range(self.max_candidate_num): # Batch encoder inputs are just re-indexed docid_inputs. batch_docid_inputs.append( np.array([ docid_inputs[batch_idx][length_idx] for batch_idx in range(local_batch_size) ], dtype=np.float32)) # Batch decoder inputs are re-indexed decoder_inputs, we create # labels. batch_labels.append( np.array([ labels[batch_idx][length_idx] for batch_idx in range(local_batch_size) ], dtype=np.float32)) # Create input feed map input_feed = {} input_feed[self.model.letor_features.name] = np.array(letor_features) for l in range(self.max_candidate_num): input_feed[self.model.docid_inputs[l].name] = batch_docid_inputs[l] input_feed[self.model.labels[l].name] = batch_labels[l] input_feed[self.model.winners.name] = np.zeros(local_batch_size) # Simulate online environment and collect clicks. input_feed = self.simulate_clicks_online(input_feed, check_validation) # Create info_map to store other information info_map = { 'rank_list_idxs': rank_list_idxs, 'input_list': docid_inputs, 'click_list': labels, 'letor_features': letor_features } self.global_batch_count += 1 if self.hparams.dynamic_bias_eta_change != 0: if self.global_batch_count % self.hparams.dynamic_bias_step_interval == 0: self.click_model.eta += self.hparams.dynamic_bias_eta_change self.click_model.setExamProb(self.click_model.eta) print('Dynamically change bias severity eta to %.3f' % self.click_model.eta) return input_feed, info_map def get_next_batch(self, index, data_set, check_validation=False): """Get the next batch of data from a specific index, prepare for step. Typically used for validation. To feed data in step(..) it must be a list of batch-major vectors, while data here contains single length-major cases. So the main logic of this function is to re-index data cases to be in the proper format for feeding. Args: index: the index of the data before which we will use to create the data batch. data_set: (Raw_data) The dataset used to build the input layer. check_validation: (bool) Set True to ignore data with no positive labels. Returns: input_feed: a feed dictionary for the next step info_map: a dictionary contain some basic information about the batch (for debugging). """ if len(data_set.initial_list[0]) < self.rank_list_size: raise ValueError( "Input ranklist length must be no less than the required list size," " %d != %d." % (len(data_set.initial_list[0]), self.rank_list_size)) docid_inputs, letor_features, labels = [], [], [] num_remain_data = len(data_set.initial_list) - index for offset in range(min(self.batch_size, num_remain_data)): i = index + offset self.prepare_true_labels_with_index(data_set, i, docid_inputs, letor_features, labels, check_validation) local_batch_size = len(docid_inputs) letor_features_length = len(letor_features) for i in range(local_batch_size): for j in range(self.max_candidate_num): if docid_inputs[i][j] < 0: docid_inputs[i][j] = letor_features_length batch_docid_inputs = [] batch_labels = [] for length_idx in range(self.max_candidate_num): # Batch encoder inputs are just re-indexed docid_inputs. batch_docid_inputs.append( np.array([ docid_inputs[batch_idx][length_idx] for batch_idx in range(local_batch_size) ], dtype=np.float32)) # Batch decoder inputs are re-indexed decoder_inputs, we create # weights. batch_labels.append( np.array([ labels[batch_idx][length_idx] for batch_idx in range(local_batch_size) ], dtype=np.float32)) # Create input feed map input_feed = {} input_feed[self.model.letor_features.name] = np.array(letor_features) for l in range(self.max_candidate_num): input_feed[self.model.docid_inputs[l].name] = batch_docid_inputs[l] input_feed[self.model.labels[l].name] = batch_labels[l] input_feed[self.model.winners.name] = np.zeros(local_batch_size) # Simulate online environment and collect clicks. input_feed = self.simulate_clicks_online(input_feed, check_validation) # Create others_map to store other information others_map = { 'input_list': docid_inputs, 'click_list': labels, } return input_feed, others_map def get_data_by_index(self, data_set, index, check_validation=False): """Get one data from the specified index, prepare for step. Args: data_set: (Raw_data) The dataset used to build the input layer. index: the index of the data check_validation: (bool) Set True to ignore data with no positive labels. Returns: The triple (docid_inputs, decoder_inputs, target_weights) for the constructed batch that has the proper format to call step(...) later. """ if len(data_set.initial_list[0]) < self.rank_list_size: raise ValueError( "Input ranklist length must be no less than the required list size," " %d != %d." % (len(data_set.initial_list[0]), self.rank_list_size)) docid_inputs, letor_features, labels = [], [], [] i = index self.prepare_true_labels_with_index(data_set, i, docid_inputs, letor_features, labels, check_validation) letor_features_length = len(letor_features) for j in range(self.max_candidate_num): if docid_inputs[-1][j] < 0: docid_inputs[-1][j] = letor_features_length batch_docid_inputs = [] batch_labels = [] for length_idx in range(self.max_candidate_num): # Batch encoder inputs are just re-indexed docid_inputs. batch_docid_inputs.append( np.array([ docid_inputs[batch_idx][length_idx] for batch_idx in range(1) ], dtype=np.float32)) # Batch decoder inputs are re-indexed decoder_inputs, we create # weights. batch_labels.append( np.array( [labels[batch_idx][length_idx] for batch_idx in range(1)], dtype=np.float32)) # Create input feed map input_feed = {} input_feed[self.model.letor_features.name] = np.array(letor_features) for l in range(self.max_candidate_num): input_feed[self.model.docid_inputs[l].name] = batch_docid_inputs[l] input_feed[self.model.labels[l].name] = batch_labels[l] input_feed[self.model.winners.name] = np.zeros(local_batch_size) # Simulate online environment and collect clicks. input_feed = self.simulate_clicks_online(input_feed, check_validation) # Create others_map to store other information others_map = { 'input_list': docid_inputs, 'click_list': labels, } return input_feed, others_map