def save_smat(features, ft_pt): """ save features to disk in SMAT format :param features: the matrix of features :param ft_pt: features file path :return: none """ (row_num, col_num) = features.shape data = features.data indice = features.indices indptr = features.indptr f = open(ft_pt, 'w') f.write("%d %d\n" % (row_num, col_num)) ind_indptr = 1 begin_line = True for ind_data in range(len(data)): while ind_data == indptr[ind_indptr]: f.write('\n') begin_line = True ind_indptr += 1 if (data[ind_data] < 1e-12) and (data[ind_data] > -1e-12): continue if (not begin_line) and (ind_data != indptr[ind_indptr - 1]): f.write(' ') f.write("%d:%s" % (indice[ind_data], data[ind_data])) begin_line = False while ind_indptr < len(indptr): f.write("\n") ind_indptr += 1 LogUtil.log("INFO", "save smat feature file done (%s)" % ft_pt) f.close()
def load_all(feature_pt, feature_names, rawset_name, will_save=False): index_begin = 0 features = None for index in reversed(range(1, len(feature_names))): f_names_s = '|'.join( feature_names[0:index + 1]) + '|' + rawset_name f_names_md5 = hashlib.md5(f_names_s).hexdigest() if isfile('%s/md5_%s.smat.npz' % (feature_pt, f_names_md5)): index_begin = index features = Feature.load('%s/md5_%s.smat' % (feature_pt, f_names_md5)) break LogUtil.log( 'INFO', 'load %s features [%s, %s)' % (rawset_name, feature_names[0], feature_names[index_begin])) if 1 > index_begin: features = Feature.load( '%s/%s.%s.smat' % (feature_pt, feature_names[0], rawset_name)) for index in range(index_begin + 1, len(feature_names)): features = Feature.merge_col( features, Feature.load('%s/%s.%s.smat' % (feature_pt, feature_names[index], rawset_name))) features = features.tocsr() if will_save and (index_begin < len(feature_names) - 1): f_names_s = '|'.join(feature_names) + '|' + rawset_name f_names_md5 = hashlib.md5(f_names_s).hexdigest() Feature.save_npz(features, '%s/md5_%s.smat' % (feature_pt, f_names_md5)) return features
def load_smat(ft_fp): """ load features from disk, the format: row_num col_num f1_index:f1_value f2_index:f2_value ... """ data = [] indice = [] indptr = [0] f = open(ft_fp) [row_num, col_num] = [int(num) for num in f.readline().strip().split()] for line in f: line = line.strip() subs = line.split() for sub in subs: [f_index, f_value] = sub.split(":") f_index = int(f_index) f_value = float(f_value) data.append(f_value) indice.append(f_index) indptr.append(len(data)) f.close() features = csr_matrix((data, indice, indptr), shape=(row_num, col_num), dtype=float) LogUtil.log("INFO", "load smat feature file done (%s)" % ft_fp) return features
def save_smat(features, ft_pt): """ save features to disk in SMAT format :param features: the matrix of features :param ft_pt: features file path :return: none """ (row_num, col_num) = features.shape data = features.data indice = features.indices indptr = features.indptr f = open(ft_pt, 'w') f.write("%d %d\n" % (row_num, col_num)) ind_indptr = 1 begin_line = True for ind_data in range(len(data)): while ind_data == indptr[ind_indptr]: f.write('\n') begin_line = True ind_indptr += 1 if (data[ind_data] < 1e-12) and (data[ind_data] > -1e-12): continue if (not begin_line) and (ind_data != indptr[ind_indptr - 1]): f.write(' ') f.write("%d:%s" % (indice[ind_data], data[ind_data])) begin_line = False while ind_indptr < len(indptr): f.write("\n") ind_indptr += 1 LogUtil.log("INFO", "save smat feature file done (%s)" % ft_pt) f.close()
def load_npz(ft_fp): loader = np.load('%s.npz' % ft_fp) features = csr_matrix( (loader['data'], loader['indices'], loader['indptr']), shape=loader['shape']) LogUtil.log("INFO", "load npz feature file done (%s)" % ft_fp) return features
def load_smat(ft_fp): """ load features from disk, the format: row_num col_num f1_index:f1_value f2_index:f2_value ... """ data = [] indice = [] indptr = [0] f = open(ft_fp) [row_num, col_num] = [int(num) for num in f.readline().strip().split()] for line in f: line = line.strip() subs = line.split() for sub in subs: [f_index, f_value] = sub.split(":") f_index = int(f_index) f_value = float(f_value) data.append(f_value) indice.append(f_index) indptr.append(len(data)) f.close() features = csr_matrix((data, indice, indptr), shape=(row_num, col_num), dtype=float) LogUtil.log("INFO", "load smat feature file done (%s)" % ft_fp) return features
def init_powerful_word_dside(pword, thresh_num, thresh_rate): pword_dside = [] pword = filter(lambda x: x[1][0] * x[1][5] >= thresh_num, pword) pword_sort = sorted(pword, key=lambda d: d[1][6], reverse=True) pword_dside.extend(map(lambda x: x[0], filter(lambda x: x[1][6] >= thresh_rate, pword_sort))) LogUtil.log('INFO', 'Double side power words(%d): %s' % (len(pword_dside), str(pword_dside))) return pword_dside
def load_all(feature_pt, feature_names, rawset_name, will_save=False): index_begin = 0 features = None for index in reversed(range(1, len(feature_names))): f_names_s = '|'.join(feature_names[0:index + 1]) + '|' + rawset_name f_names_md5 = hashlib.md5(f_names_s).hexdigest() if isfile('%s/md5_%s.smat.npz' % (feature_pt, f_names_md5)): index_begin = index features = Feature.load('%s/md5_%s.smat' % (feature_pt, f_names_md5)) break LogUtil.log('INFO', 'load %s features [%s, %s)' % (rawset_name, feature_names[0], feature_names[index_begin])) if 1 > index_begin: features = Feature.load('%s/%s.%s.smat' % (feature_pt, feature_names[0], rawset_name)) for index in range(index_begin + 1, len(feature_names)): features = Feature.merge_col(features, Feature.load( '%s/%s.%s.smat' % (feature_pt, feature_names[index], rawset_name))) features = features.tocsr() if will_save and (index_begin < len(feature_names) - 1): f_names_s = '|'.join(feature_names) + '|' + rawset_name f_names_md5 = hashlib.md5(f_names_s).hexdigest() Feature.save_npz(features, '%s/md5_%s.smat' % (feature_pt, f_names_md5)) return features
def __generate_index(self, row_num): index_all = [list()] * self.cv_num for i in range(row_num): index_all[int(random.random() * self.cv_num)].append(i) for i in range(self.cv_num): LogUtil.log( 'INFO', 'generate cv index, size(part%d)=%d' % (i, len(index_all[i]))) index_pt = self.config.get('DEFAULT', 'index_pt') for i in range(self.cv_num): fold_id = i # train fp = '%s/cv_tag%s_n%d_f%d_train.%s.index' % ( index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(fp, list(), 'w') for j in range(self.cv_num - 2): part_id = (i + j) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'a') # valid fp = '%s/cv_tag%s_n%d_f%d_valid.%s.index' % ( index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) part_id = (fold_id + self.cv_num - 2) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'w') # test fp = '%s/cv_tag%s_n%d_f%d_test.%s.index' % ( index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) part_id = (fold_id + self.cv_num - 1) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'w')
def extract(self, data_set_name, part_num=1, part_id=0): """ Extract the feature from original data set :param data_set_name: name of data set :param part_num: number of partitions of data :param part_id: partition ID which will be extracted :return: """ # load data set from disk data = pd.read_csv('%s/%s.csv' % (self.config.get('DEFAULT', 'source_pt'), data_set_name)).fillna(value="") begin_id = int(1. * len(data) / part_num * part_id) end_id = int(1. * len(data) / part_num * (part_id + 1)) # set feature file path feature_pt = self.config.get('DEFAULT', 'feature_pt') if 1 == part_num: self.data_feature_fp = '%s/%s.%s.smat' % (feature_pt, self.feature_name, data_set_name) else: self.data_feature_fp = '%s/%s.%s.smat.%03d_%03d' % (feature_pt, self.feature_name, data_set_name, part_num, part_id) feature_file = open(self.data_feature_fp, 'w') feature_file.write('%d %d\n' % (end_id - begin_id, int(self.get_feature_num()))) # extract feature for index, row in data[begin_id:end_id].iterrows(): feature = self.extract_row(row) Feature.save_feature(feature, feature_file) feature_file.close() LogUtil.log('INFO', 'save features (%s, %s, %d, %d) done' % (self.feature_name, data_set_name, part_num, part_id))
def init_powerful_word_oside(pword, thresh_num, thresh_rate): pword_oside = [] pword = filter(lambda x: x[1][0] * x[1][3] >= thresh_num, pword) pword_oside.extend( map(lambda x: x[0], filter(lambda x: x[1][4] >= thresh_rate, pword))) LogUtil.log('INFO', 'One side power words(%d): %s' % ( len(pword_oside), str(pword_oside))) return pword_oside
def load_npz(ft_fp): loader = np.load('%s.npz' % ft_fp) features = csr_matrix((loader['data'], loader['indices'], loader['indptr']), shape=loader['shape']) LogUtil.log("INFO", "load npz feature file done (%s)" % ft_fp) return features
def __unlock(self): lock_name = self.config.get('MODEL', 'lock_name') lock_pt = self.config.get('MODEL', 'lock_pt') lock_fp = '%s/%s.lock' % (lock_pt, lock_name) if isfile(lock_fp): os.remove(lock_fp) LogUtil.log('INFO', 'delete lock, lock_name=%s' % lock_name) else: LogUtil.log('WARNING', 'missing lock, lock_name=%s' % lock_name)
def get_labels(df): """ Get labels of data set :param df: original data set :return: label list of data set """ labels = df['is_duplicate'].tolist() LogUtil.log("INFO", "num(1)=%d, num(0)=%d" % (sum(labels), len(labels) - sum(labels))) return labels
def __unlock(self): lock_name = self.config.get('MODEL', 'lock_name') lock_pt = self.config.get('MODEL', 'lock_pt') lock_fp = '%s/%s.lock' % (lock_pt, lock_name) if isfile(lock_fp): os.remove(lock_fp) LogUtil.log('INFO', 'delete lock, lock_name=%s' % lock_name) else: LogUtil.log('WARNING', 'missing lock, lock_name=%s' % lock_name)
def init_powerful_word_oside(pword, thresh_num, thresh_rate): pword_oside = [] pword = filter(lambda x: x[1][0] * x[1][3] >= thresh_num, pword) pword_oside.extend( map(lambda x: x[0], filter(lambda x: x[1][4] >= thresh_rate, pword))) LogUtil.log( 'INFO', 'One side power words(%d): %s' % (len(pword_oside), str(pword_oside))) return pword_oside
def generate_powerful_word(data, subset_indexs): """ 计算数据中词语的影响力,格式如下: 词语 --> [0. 出现语句对数量,1. 出现语句对比例,2. 正确语句对比例,3. 单侧语句对比例,4. 单侧语句对正确比例,5. 双侧语句对比例,6. 双侧语句对正确比例] """ words_power = {} train_subset_data = data.iloc[subset_indexs, :] for index, row in train_subset_data.iterrows(): label = int(row['is_duplicate']) q1_words = str(row['question1']).lower().split() q2_words = str(row['question2']).lower().split() all_words = set(q1_words + q2_words) q1_words = set(q1_words) q2_words = set(q2_words) for word in all_words: if word not in words_power: words_power[word] = [0. for i in range(7)] # 计算出现语句对数量 words_power[word][0] += 1. words_power[word][1] += 1. if ((word in q1_words) and (word not in q2_words)) or ((word not in q1_words) and (word in q2_words)): # 计算单侧语句数量 words_power[word][3] += 1. if 0 == label: # 计算正确语句对数量 words_power[word][2] += 1. # 计算单侧语句正确比例 words_power[word][4] += 1. if (word in q1_words) and (word in q2_words): # 计算双侧语句数量 words_power[word][5] += 1. if 1 == label: # 计算正确语句对数量 words_power[word][2] += 1. # 计算双侧语句正确比例 words_power[word][6] += 1. for word in words_power: # 计算出现语句对比例 words_power[word][1] /= len(subset_indexs) # 计算正确语句对比例 words_power[word][2] /= words_power[word][0] # 计算单侧语句对正确比例 if words_power[word][3] > 1e-6: words_power[word][4] /= words_power[word][3] # 计算单侧语句对比例 words_power[word][3] /= words_power[word][0] # 计算双侧语句对正确比例 if words_power[word][5] > 1e-6: words_power[word][6] /= words_power[word][5] # 计算双侧语句对比例 words_power[word][5] /= words_power[word][0] sorted_words_power = sorted(words_power.iteritems(), key=lambda d: d[1][0], reverse=True) LogUtil.log("INFO", "power words calculation done, len(words_power)=%d" % len(sorted_words_power)) return sorted_words_power
def merge_col(features_1, features_2): """ merge features made split by column :param features_1: the first part of features :param features_2: the second part of features :return: feature matrix """ features = hstack([features_1, features_2]) (row_num, col_num) = features.shape LogUtil.log("INFO", "merge col done, shape=(%d,%d)" % (row_num, col_num)) return features
def extract(self): version = self.config.get('INFO', 'version') cv_num = self.config.get('INFO', 'cv_num') offline_rawset_name = self.config.get('MODEL', 'offline_rawset_name') index_fp = self.config.get('DIRECTORY', 'feature_pt') feature_name = '%s_%s' % (self.__class__.__name__, version) # load prediction of offline tests offline_test_pred_all_fp = '%s/pred/cv_n%d_test.%s.pred' % ( self.config.get('DIRECTORY', 'out_pt'), cv_num, offline_rawset_name) offline_test_pred_all_origin = PostProcessor.read_result_list( offline_test_pred_all_fp) offline_test_pred_all = [0] * len(offline_test_pred_all_origin) # load index of offline tests offline_test_index_all = list() for fold_id in range(cv_num): offline_test_indexs_fp = '%s/cv_n%d_f%d_test.%s.index' % ( index_fp, cv_num, fold_id, offline_rawset_name) offline_test_indexs = Feature.load_index(offline_test_indexs_fp) offline_test_index_all.extend(offline_test_indexs) for index in range(len(offline_test_pred_all)): offline_test_pred_all[offline_test_index_all[ index]] = offline_test_pred_all_origin[index] # load prediction of online data set online_preds = list() for fold_id in range(cv_num): online_pred_fp = '%s/cv_n%d_f%d_online.%s.pred' % ( self.config.get('DIRECTORY', 'pred_pt'), cv_num, fold_id, self.config.get('MODEL', 'online_test_rawset_name')) online_pred_one = PostProcessor.read_result_list(online_pred_fp) online_preds.append(online_pred_one) # sample for online prediction online_pred = [] for i in range(len(online_preds[0])): cv_id = int(random.random() * cv_num) online_pred.append(online_preds[cv_id][i]) offline_pred = [[fv] for fv in offline_test_pred_all] online_pred = [[fv] for fv in online_pred] # directory of features feature_pt = self.config.get('DIRECTORY', 'feature_pt') train_feature_fp = '%s/%s.train.smat' % (feature_pt, feature_name) test_feature_fp = '%s/%s.test.smat' % (feature_pt, feature_name) train_features = sparse.csr_matrix(np.array(offline_pred)) Feature.save_smat(train_features, train_feature_fp) LogUtil.log('INFO', 'save train features (%s) done' % feature_name) test_features = sparse.csr_matrix(np.array(online_pred)) Feature.save_smat(test_features, test_feature_fp) LogUtil.log('INFO', 'save test features (%s) done' % feature_name)
def init_powerful_word_dside(pword, thresh_num, thresh_rate): pword_dside = [] pword = filter(lambda x: x[1][0] * x[1][5] >= thresh_num, pword) pword_sort = sorted(pword, key=lambda d: d[1][6], reverse=True) pword_dside.extend( map(lambda x: x[0], filter(lambda x: x[1][6] >= thresh_rate, pword_sort))) LogUtil.log( 'INFO', 'Double side power words(%d): %s' % (len(pword_dside), str(pword_dside))) return pword_dside
def stat_dul_question(df): """ Make statistics to duplication of questions :param df: original data set :return: none """ questions = df['question1'].tolist() + df['question2'].tolist() len_questions = len(questions) len_uniq_questions = len(set(questions)) LogUtil.log("INFO", "len(questions)=%d, len(unique_questions)=%d, rate=%f" % ( len_questions, len_uniq_questions, 1.0 * len_uniq_questions / len_questions))
def init_tfidf(self): train_data = pd.read_csv('%s/train.csv' % self.config.get('DIRECTORY', 'origin_pt')).fillna(value="") # [:100] test_data = pd.read_csv('%s/test.csv' % self.config.get('DIRECTORY', 'origin_pt')).fillna(value="") # [:100] tfidf = TfidfVectorizer(stop_words='english', ngram_range=(1, 1)) tfidf_txt = pd.Series( train_data['question1'].tolist() + train_data['question2'].tolist() + test_data['question1'].tolist() + test_data['question2'].tolist()).astype(str) tfidf.fit_transform(tfidf_txt) LogUtil.log("INFO", "init tfidf done ") return tfidf
def merge_col(features_1, features_2): """ merge features made split by column :param features_1: the first part of features :param features_2: the second part of features :return: feature matrix """ features = hstack([features_1, features_2]) (row_num, col_num) = features.shape LogUtil.log("INFO", "merge col done, shape=(%d,%d)" % (row_num, col_num)) return features
def __lock(self): lock_name = self.config.get('MODEL', 'lock_name') lock_time = self.config.getint('MODEL', 'lock_time') lock_pt = self.config.get('MODEL', 'lock_pt') if '' != lock_name: lock_fp = '%s/%s.lock' % (lock_pt, lock_name) while isfile(lock_fp): LogUtil.log('INFO', 'model is running, lock_name=%s, waiting %d ...' % (lock_name, lock_time)) time.sleep(lock_time) f = open(lock_fp, 'w') f.close() LogUtil.log('INFO', 'generate lock, lock_name=%s' % lock_name)
def generate_graph_clique(G): n2clique = {} cliques = [] for clique in nx.find_cliques(G): for n in clique: if n not in n2clique: n2clique[n] = [] n2clique[n].append(len(cliques)) cliques.append(clique) LogUtil.log('INFO', 'init graph cliques done, len(cliques)=%d' % len(cliques)) return n2clique, cliques
def generate_graph_cc(G): n2cc = {} ccs = [] for cc in nx.connected_components(G): for n in cc: if n in n2cc: LogUtil.log('WARNING', '%d already in n2cc(=%d)' % (n, n2cc[n])) n2cc[n] = len(ccs) ccs.append(cc) LogUtil.log('INFO', 'init graph cc done, len(cliques)=%d' % len(ccs)) return n2cc, ccs
def generate_idf(data_fp): data = csv.reader(data_fp) idf = {} for index, row in data.iterrows(): words = str(row['question']).strip().split() if WordEmbedding.to_lower else str( row['question']).lower().strip().split() for word in words: idf[word] = idf.get(word, 0) + 1 num_docs = len(data) for word in idf: idf[word] = math.log(num_docs / (idf[word] + 1.)) / math.log(2.) LogUtil.log("INFO", "IDF calculation done, len(idf)=%d" % len(idf)) return idf
def get_qid2question(df): """ Get map(qid, question) :param df: original data set :return: map(qid, question) """ qid2question = {} qids = df['qid1'].tolist() + df['qid2'].tolist() questions = df['question1'].tolist() + df['question2'].tolist() for ind in range(len(qids)): qid2question[qids[ind]] = questions[ind] LogUtil.log("INFO", "len(qids)=%d, len(unique_qids)=%d" % (len(qids), len(qid2question))) return qid2question
def save_npz(features, ft_fp): """ save features to disk in binary format :param features: :param ft_fp: :return: """ np.savez(ft_fp, data=features.data, indices=features.indices, indptr=features.indptr, shape=features.shape) LogUtil.log('INFO', 'save npz feature file done (%s)' % ft_fp)
def save_npz(features, ft_fp): """ save features to disk in binary format :param features: :param ft_fp: :return: """ np.savez(ft_fp, data=features.data, indices=features.indices, indptr=features.indptr, shape=features.shape) LogUtil.log('INFO', 'save npz feature file done (%s)' % ft_fp)
def __lock(self): lock_name = self.config.get('MODEL', 'lock_name') lock_time = self.config.getint('MODEL', 'lock_time') lock_pt = self.config.get('MODEL', 'lock_pt') if '' != lock_name: lock_fp = '%s/%s.lock' % (lock_pt, lock_name) while isfile(lock_fp): LogUtil.log( 'INFO', 'model is running, lock_name=%s, waiting %d ...' % (lock_name, lock_time)) time.sleep(lock_time) f = open(lock_fp, 'w') f.close() LogUtil.log('INFO', 'generate lock, lock_name=%s' % lock_name)
def generate_idf(data_fp): data = csv.reader(data_fp) idf = {} for index, row in data.iterrows(): words = str(row['question']).strip().split( ) if WordEmbedding.to_lower else str( row['question']).lower().strip().split() for word in words: idf[word] = idf.get(word, 0) + 1 num_docs = len(data) for word in idf: idf[word] = math.log(num_docs / (idf[word] + 1.)) / math.log(2.) LogUtil.log("INFO", "IDF calculation done, len(idf)=%d" % len(idf)) return idf
def merge_file(feature_pt, feature_name, data_set_name, part_num): features = None for part_id in range(part_num): features_part_fp = '%s/%s.%s.smat.%03d_%03d' % (feature_pt, feature_name, data_set_name, part_num, part_id) features_part = Feature.load(features_part_fp) if features is None: features = features_part else: features = Feature.merge_row(features, features_part) features_fp = '%s/%s.%s.smat' % (feature_pt, feature_name, data_set_name) Feature.save_smat(features, features_fp) LogUtil.log('INFO', 'merge features (%s, %s, %d) done' % (feature_name, data_set_name, part_num))
def init_tfidf(self): train_data = pd.read_csv( '%s/train.csv' % self.config.get('DEFAULT', 'origin_pt')).fillna( value="") # [:100] test_data = pd.read_csv( '%s/test.csv' % self.config.get('DEFAULT', 'origin_pt')).fillna( value="") # [:100] tfidf = TfidfVectorizer(stop_words='english', ngram_range=(1, 1)) tfidf_txt = pd.Series(train_data['question1'].tolist() + train_data['question2'].tolist() + test_data['question1'].tolist() + test_data['question2'].tolist()).astype(str) tfidf.fit_transform(tfidf_txt) LogUtil.log("INFO", "init tfidf done ") return tfidf
def generate_cv_subset_index(cf, argv): """ Generate index used for 5-fold cross validation :param cf: configuration file :param argv: parameter list :return: none """ tag = argv[0] cv_num = 5 cv_rawset_name = 'train_with_swap' train_data_size = 404290 index_all = [] for i in range(cv_num): index_all.append([]) for i in range(train_data_size): index_all[int(random.random() * cv_num)].append(i) for i in range(cv_num): LogUtil.log('INFO', 'size(part%d)=%d' % (i, len(index_all[i]))) index_fp = cf.get('DEFAULT', 'feature_index_pt') for i in range(cv_num): fold_id = i # train fp = '%s/cv_tag%s_n%d_f%d_train.%s.index' % ( index_fp, tag, cv_num, fold_id, cv_rawset_name) for j in range(cv_num - 2): part_id = (i + j) % cv_num DataUtil.save_vector(fp, index_all[part_id], 'a') for j in range(cv_num - 2): part_id = (i + j) % cv_num DataUtil.save_vector( fp, [index + train_data_size for index in index_all[part_id]], 'a') # valid fp = '%s/cv_tag%s_n%d_f%d_valid.%s.index' % ( index_fp, tag, cv_num, fold_id, cv_rawset_name) part_id = (fold_id + cv_num - 2) % cv_num DataUtil.save_vector(fp, index_all[part_id], 'w') # test fp = '%s/cv_tag%s_n%d_f%d_test.%s.index' % ( index_fp, tag, cv_num, fold_id, cv_rawset_name) part_id = (fold_id + cv_num - 1) % cv_num DataUtil.save_vector(fp, index_all[part_id], 'w')
def merge_file(feature_pt, feature_name, data_set_name, part_num): features = None for part_id in range(part_num): features_part_fp = '%s/%s.%s.smat.%03d_%03d' % ( feature_pt, feature_name, data_set_name, part_num, part_id) features_part = Feature.load(features_part_fp) if features is None: features = features_part else: features = Feature.merge_row(features, features_part) features_fp = '%s/%s.%s.smat' % (feature_pt, feature_name, data_set_name) Feature.save_smat(features, features_fp) LogUtil.log( 'INFO', 'merge features (%s, %s, %d) done' % (feature_name, data_set_name, part_num))
def __init_out_dir(self): # generate output tag self.out_tag = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) self.config.set('DIRECTORY', str(self.out_tag)) # generate output directory out_pt = self.config.get('DIRECTORY', 'out_pt') out_pt_exists = os.path.exists(out_pt) if out_pt_exists: LogUtil.log("ERROR", 'out path (%s) already exists ' % out_pt) return else: os.mkdir(out_pt) os.mkdir(self.config.get('DIRECTORY', 'pred_pt')) os.mkdir(self.config.get('DIRECTORY', 'model_pt')) os.mkdir(self.config.get('DIRECTORY', 'fault_pt')) os.mkdir(self.config.get('DIRECTORY', 'conf_pt')) os.mkdir(self.config.get('DIRECTORY', 'score_pt')) LogUtil.log('INFO', 'out path (%s) created ' % out_pt) # save config self.config.write(open(self.config.get('DIRECTORY', 'conf_pt') + 'featwheel.conf', 'w'))
def balance_index(indexs, labels, positive_rate): """ balance indexs to adjust the positive rate :param indexs: index vector to sample raw data set :param labels: label vector of raw data set :param positive_rate: positive rate :return: index vector after balanced """ if positive_rate < 1e-6 or positive_rate > 1. - 1e-6: return indexs pos_indexs = [index for index in indexs if labels[index] == 1.] neg_indexs = [index for index in indexs if labels[index] == 0.] origin_rate = 1.0 * len(pos_indexs) / len(indexs) LogUtil.log( "INFO", "original: len(pos)=%d, len(neg)=%d, rate=%.2f%%" % (len(pos_indexs), len(neg_indexs), 100.0 * origin_rate)) if origin_rate < positive_rate: pos_indexs, neg_indexs = neg_indexs, pos_indexs origin_rate = 1.0 - origin_rate positive_rate = 1.0 - positive_rate LogUtil.log("INFO", "increase postive instances ...") else: LogUtil.log("INFO", "increase negtive instances ...") k = (1. - positive_rate) * origin_rate / positive_rate / (1 - origin_rate) LogUtil.log("INFO", "k=%.4f" % k) balance_indexs = pos_indexs while k > 1e-6: if k > 1. - 1e-6: balance_indexs.extend(neg_indexs) else: balance_indexs.extend( random.sample(neg_indexs, int(k * len(neg_indexs)))) k -= 1. pos_indexs = [index for index in balance_indexs if labels[index] == 1.] neg_indexs = [index for index in balance_indexs if labels[index] == 0.] balanced_rate = 1.0 * len(pos_indexs) / len(balance_indexs) LogUtil.log( "INFO", "balanced: len(pos)=%d, len(neg)=%d, rate=%.2f%%" % (len(pos_indexs), len(neg_indexs), 100.0 * balanced_rate)) return balance_indexs
def init_idf(data): idf = {} q_set = set() for index, row in data.iterrows(): q1 = str(row['question1']) q2 = str(row['question2']) if q1 not in q_set: q_set.add(q1) words = q1.lower().split() for word in words: idf[word] = idf.get(word, 0) + 1 if q2 not in q_set: q_set.add(q2) words = q2.lower().split() for word in words: idf[word] = idf.get(word, 0) + 1 num_docs = len(data) for word in idf: idf[word] = math.log(num_docs / (idf[word] + 1.)) / math.log(2.) LogUtil.log("INFO", "idf calculation done, len(idf)=%d" % len(idf)) return idf
def init_idf(data): idf = {} q_set = set() for index, row in data.iterrows(): q1 = str(row['question1']) q2 = str(row['question2']) if q1 not in q_set: q_set.add(q1) words = q1.lower().split() for word in words: idf[word] = idf.get(word, 0) + 1 if q2 not in q_set: q_set.add(q2) words = q2.lower().split() for word in words: idf[word] = idf.get(word, 0) + 1 num_docs = len(data) for word in idf: idf[word] = math.log(num_docs / (idf[word] + 1.)) / math.log(2.) LogUtil.log("INFO", "idf calculation done, len(idf)=%d" % len(idf)) return idf
def balance_index(indexs, labels, positive_rate): """ balance indexs to adjust the positive rate :param indexs: index vector to sample raw data set :param labels: label vector of raw data set :param positive_rate: positive rate :return: index vector after balanced """ if positive_rate < 1e-6 or positive_rate > 1. - 1e-6: return indexs pos_indexs = [index for index in indexs if labels[index] == 1.] neg_indexs = [index for index in indexs if labels[index] == 0.] origin_rate = 1.0 * len(pos_indexs) / len(indexs) LogUtil.log("INFO", "original: len(pos)=%d, len(neg)=%d, rate=%.2f%%" % ( len(pos_indexs), len(neg_indexs), 100.0 * origin_rate)) if origin_rate < positive_rate: pos_indexs, neg_indexs = neg_indexs, pos_indexs origin_rate = 1.0 - origin_rate positive_rate = 1.0 - positive_rate LogUtil.log("INFO", "increase postive instances ...") else: LogUtil.log("INFO", "increase negtive instances ...") k = (1. - positive_rate) * origin_rate / positive_rate / (1 - origin_rate) LogUtil.log("INFO", "k=%.4f" % k) balance_indexs = pos_indexs while k > 1e-6: if k > 1. - 1e-6: balance_indexs.extend(neg_indexs) else: balance_indexs.extend(random.sample(neg_indexs, int(k * len(neg_indexs)))) k -= 1. pos_indexs = [index for index in balance_indexs if labels[index] == 1.] neg_indexs = [index for index in balance_indexs if labels[index] == 0.] balanced_rate = 1.0 * len(pos_indexs) / len(balance_indexs) LogUtil.log("INFO", "balanced: len(pos)=%d, len(neg)=%d, rate=%.2f%%" % ( len(pos_indexs), len(neg_indexs), 100.0 * balanced_rate)) return balance_indexs
def __generate_index(self, row_num): index_all = [list()] * self.cv_num for i in range(row_num): index_all[int(random.random() * self.cv_num)].append(i) for i in range(self.cv_num): LogUtil.log('INFO', 'generate cv index, size(part%d)=%d' % (i, len(index_all[i]))) index_pt = self.config.get('DEFAULT', 'index_pt') for i in range(self.cv_num): fold_id = i # train fp = '%s/cv_tag%s_n%d_f%d_train.%s.index' % (index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(fp, list(), 'w') for j in range(self.cv_num - 2): part_id = (i + j) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'a') # valid fp = '%s/cv_tag%s_n%d_f%d_valid.%s.index' % (index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) part_id = (fold_id + self.cv_num - 2) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'w') # test fp = '%s/cv_tag%s_n%d_f%d_test.%s.index' % (index_pt, self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) part_id = (fold_id + self.cv_num - 1) % self.cv_num DataUtil.save_vector(fp, index_all[part_id], 'w')
def __init_out_dir(self): # generate output tag self.out_tag = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) self.config.set('DIRECTORY', str(self.out_tag)) # generate output directory out_pt = self.config.get('DIRECTORY', 'out_pt') out_pt_exists = os.path.exists(out_pt) if out_pt_exists: LogUtil.log("ERROR", 'out path (%s) already exists ' % out_pt) return else: os.mkdir(out_pt) os.mkdir(self.config.get('DIRECTORY', 'pred_pt')) os.mkdir(self.config.get('DIRECTORY', 'model_pt')) os.mkdir(self.config.get('DIRECTORY', 'fault_pt')) os.mkdir(self.config.get('DIRECTORY', 'conf_pt')) os.mkdir(self.config.get('DIRECTORY', 'score_pt')) LogUtil.log('INFO', 'out path (%s) created ' % out_pt) # save config self.config.write( open( self.config.get('DIRECTORY', 'conf_pt') + 'featwheel.conf', 'w'))
def run_offline(self): LogUtil.log('INFO', 'cv_tag(%s)' % self.cv_tag) # load feature matrix offline_features = Feature.load_all( self.config.get('DIRECTORY', 'feature_pt'), self.config.get('FEATURE', 'feature_selected').split(), self.config.get('MODEL', 'offline_rawset_name'), self.config.get('FEATURE', 'will_save')) # load labels offline_labels = DataUtil.load_vector( '%s/%s.label' % (self.config.get('DIRECTORY', 'label_pt'), self.config.get('MODEL', 'offline_rawset_name')), True) # generate index file if '' == self.cv_tag: self.cv_tag = self.out_tag self.__generate_index(offline_features.shape[0]) # cross validation offline_valid_preds_all = [0.] * offline_features.shape[0] offline_test_preds_all = [0.] * offline_features.shape[0] for fold_id in range(self.cv_num): LogUtil.log('INFO', 'cross validation fold_id(%d) begin' % fold_id) # generate training data set offline_train_pos_rate = float( self.config.get('MODEL', 'train_pos_rate')) offline_train_indexs_fp = '%s/cv_tag%s_n%d_f%d_train.%s.index' % ( self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_train_indexs = DataUtil.load_vector( offline_train_indexs_fp, 'int') offline_train_features, offline_train_labels, offline_train_balanced_indexs = \ CrossValidation.__generate_data(offline_train_indexs, offline_labels, offline_features, offline_train_pos_rate) LogUtil.log('INFO', 'offline train data generation done') # generate validation data set offline_valid_pos_rate = float( self.config.get('MODEL', 'valid_pos_rate')) offline_valid_indexs_fp = '%s/cv_tag%s_n%d_f%d_valid.%s.index' % ( self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_valid_indexs = DataUtil.load_vector( offline_valid_indexs_fp, 'int') offline_valid_features, offline_valid_labels, offline_valid_balanced_indexs = \ CrossValidation.__generate_data(offline_valid_indexs, offline_labels, offline_features, offline_valid_pos_rate) LogUtil.log('INFO', 'offline valid data generation done') # generate test data set offline_test_pos_rate = float( self.config.get('MODEL', 'test_pos_rate')) offline_test_indexs_fp = '%s/cv_tag%s_n%d_f%d_test.%s.index' % ( self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_test_indexs = DataUtil.load_vector(offline_test_indexs_fp, 'int') offline_test_features, offline_test_labels, offline_test_balanced_indexs = \ CrossValidation.__generate_data(offline_test_indexs, offline_labels, offline_features, offline_test_pos_rate) LogUtil.log('INFO', 'offline test data generation done') model = Model.new(self.config.get('MODEL', 'model_name'), self.config) model_fp = self.config.get('DIRECTORY', 'model_pt') + '/cv_n%d_f%d.%s.model' % \ (self.cv_num, fold_id, self.config.get('MODEL', 'model_name')) model.save(model_fp) offline_train_preds, offline_valid_preds, offline_test_preds = model.fit( offline_train_features, offline_train_labels, offline_valid_features, offline_valid_labels, offline_test_features, offline_test_labels) offline_train_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_train_labels, offline_train_preds) offline_valid_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_valid_labels, offline_valid_preds) offline_test_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_test_labels, offline_test_preds) score_fp = '%s/%s.score' % (self.config.get( 'DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write('fold:%d\ttrain:%s\tvalid:%s\ttest:%s\n' % (fold_id, offline_train_score, offline_valid_score, offline_test_score)) score_file.close() # merge prediction results for index in range(len(offline_valid_balanced_indexs)): offline_valid_preds_all[offline_valid_balanced_indexs[ index]] = offline_valid_preds[index] for index in range(len(offline_test_balanced_indexs)): offline_test_preds_all[offline_test_balanced_indexs[ index]] = offline_test_preds[index] LogUtil.log('INFO', 'cross test fold_id(%d) done' % fold_id) # save prediction results offline_valid_preds_all_fp = '%s/cv_n%d_valid.%s.pred' % ( self.config.get('DIRECTORY', 'pred_pt'), self.cv_num, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_valid_preds_all_fp, offline_valid_preds_all, 'w') offline_test_preds_all_fp = '%s/cv_n%d_test.%s.pred' % ( self.config.get('DIRECTORY', 'pred_pt'), self.cv_num, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_test_preds_all_fp, offline_test_preds_all, 'w') # evaluate offline_valid_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_labels, offline_valid_preds_all) offline_test_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_labels, offline_test_preds_all) score_fp = '%s/%s.score' % (self.config.get('DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write('cross_validation\tvalid:%s\ttest:%s\n' % (offline_valid_score, offline_test_score)) score_file.close()
def sample_col(features, indexs): features_sampled = features[:, indexs] (row_num, col_num) = features_sampled.shape LogUtil.log("INFO", "col sample done, shape=(%d,%d)" % (row_num, col_num)) return features_sampled
def run_offline(self): # load feature matrix offline_features = Feature.load_all( self.config.get('DIRECTORY', 'feature_pt'), self.config.get('FEATURE', 'feature_selected').split(), self.config.get('MODEL', 'offline_rawset_name'), self.config.get('FEATURE', 'will_save')) # load labels offline_labels = DataUtil.load_vector( '%s/%s.label' % (self.config.get('DIRECTORY', 'label_pt'), self.config.get('MODEL', 'offline_rawset_name')), True) # generate index file if '' == self.se_tag: self.se_tag = self.out_tag self.__generate_index(offline_features.shape[0]) index_pt = self.config.get('DIRECTORY', 'index_pt') # generate training data set offline_train_pos_rate = float( self.config.get('MODEL', 'train_pos_rate')) offline_train_indexs_fp = '%s/se_tag%s_train.%s.index' % ( index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_train_indexs = DataUtil.load_vector(offline_train_indexs_fp, 'int') offline_train_features, offline_train_labels, offline_train_balanced_indexs = \ SingleExec.__generate_data(offline_train_indexs, offline_labels, offline_features, offline_train_pos_rate) LogUtil.log('INFO', 'offline train data generation done') # generate validation data set offline_valid_pos_rate = float( self.config.get('MODEL', 'valid_pos_rate')) offline_valid_indexs_fp = '%s/se_tag%s_valid.%s.index' % ( index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_valid_indexs = DataUtil.load_vector(offline_valid_indexs_fp, 'int') offline_valid_features, offline_valid_labels, offline_valid_balanced_indexs = \ SingleExec.__generate_data(offline_valid_indexs, offline_labels, offline_features, offline_valid_pos_rate) LogUtil.log('INFO', 'offline valid data generation done') # generate test data set offline_test_pos_rate = float(self.config.get('MODEL', 'test_pos_rate')) offline_test_indexs_fp = '%s/se_tag%s_test.%s.index' % ( index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_test_indexs = DataUtil.load_vector(offline_test_indexs_fp, 'int') offline_test_features, offline_test_labels, offline_test_balanced_indexs = \ SingleExec.__generate_data(offline_test_indexs, offline_labels, offline_features, offline_test_pos_rate) LogUtil.log('INFO', 'offline test data generation done') model = Model.new(self.config.get('MODEL', 'model_name'), self.config) model_fp = self.config.get( 'DIRECTORY', 'model_pt') + '/se.%s.model' % self.config.get( 'MODEL', 'model_name') model.save(model_fp) offline_train_preds, offline_valid_preds, offline_test_preds = model.fit( offline_train_features, offline_train_labels, offline_valid_features, offline_valid_labels, offline_test_features, offline_test_labels) offline_train_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_train_labels, offline_train_preds) offline_valid_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_valid_labels, offline_valid_preds) offline_test_score = Evaluator.evaluate( self.config.get('MODEL', 'evaluator_name'), offline_test_labels, offline_test_preds) score_fp = '%s/%s.score' % (self.config.get('DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write( 'single_exec\ttrain:%s\tvalid:%s\ttest:%s\n' % (offline_train_score, offline_valid_score, offline_test_score)) score_file.close() # save prediction results offline_valid_preds_fp = '%s/se_valid.%s.pred' % (self.config.get( 'DIRECTORY', 'pred_pt'), self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_valid_preds_fp, offline_valid_preds, 'w') offline_test_preds_fp = '%s/se_test.%s.pred' % (self.config.get( 'DIRECTORY', 'pred_pt'), self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_test_preds_fp, offline_test_preds, 'w')
def sample_row(features, indexs): features_sampled = features[indexs, :] (row_num, col_num) = features_sampled.shape LogUtil.log("INFO", "row sample done, shape=(%d,%d)" % (row_num, col_num)) return features_sampled
def run_offline(self): # load feature matrix offline_features = Feature.load_all(self.config.get('DIRECTORY', 'feature_pt'), self.config.get('FEATURE', 'feature_selected').split(), self.config.get('MODEL', 'offline_rawset_name'), self.config.get('FEATURE', 'will_save')) # load labels offline_labels = DataUtil.load_vector('%s/%s.label' % (self.config.get('DIRECTORY', 'label_pt'), self.config.get('MODEL', 'offline_rawset_name')), True) # generate index file if '' == self.se_tag: self.se_tag = self.out_tag self.__generate_index(offline_features.shape[0]) index_pt = self.config.get('DIRECTORY', 'index_pt') # generate training data set offline_train_pos_rate = float(self.config.get('MODEL', 'train_pos_rate')) offline_train_indexs_fp = '%s/se_tag%s_train.%s.index' % (index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_train_indexs = DataUtil.load_vector(offline_train_indexs_fp, 'int') offline_train_features, offline_train_labels, offline_train_balanced_indexs = \ SingleExec.__generate_data(offline_train_indexs, offline_labels, offline_features, offline_train_pos_rate) LogUtil.log('INFO', 'offline train data generation done') # generate validation data set offline_valid_pos_rate = float(self.config.get('MODEL', 'valid_pos_rate')) offline_valid_indexs_fp = '%s/se_tag%s_valid.%s.index' % (index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_valid_indexs = DataUtil.load_vector(offline_valid_indexs_fp, 'int') offline_valid_features, offline_valid_labels, offline_valid_balanced_indexs = \ SingleExec.__generate_data(offline_valid_indexs, offline_labels, offline_features, offline_valid_pos_rate) LogUtil.log('INFO', 'offline valid data generation done') # generate test data set offline_test_pos_rate = float(self.config.get('MODEL', 'test_pos_rate')) offline_test_indexs_fp = '%s/se_tag%s_test.%s.index' % (index_pt, self.se_tag, self.config.get('MODEL', 'offline_rawset_name')) offline_test_indexs = DataUtil.load_vector(offline_test_indexs_fp, 'int') offline_test_features, offline_test_labels, offline_test_balanced_indexs = \ SingleExec.__generate_data(offline_test_indexs, offline_labels, offline_features, offline_test_pos_rate) LogUtil.log('INFO', 'offline test data generation done') model = Model.new(self.config.get('MODEL', 'model_name'), self.config) model_fp = self.config.get('DIRECTORY', 'model_pt') + '/se.%s.model' % self.config.get('MODEL', 'model_name') model.save(model_fp) offline_train_preds, offline_valid_preds, offline_test_preds = model.fit(offline_train_features, offline_train_labels, offline_valid_features, offline_valid_labels, offline_test_features, offline_test_labels) offline_train_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_train_labels, offline_train_preds) offline_valid_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_valid_labels, offline_valid_preds) offline_test_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_test_labels, offline_test_preds) score_fp = '%s/%s.score' % (self.config.get('DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write('single_exec\ttrain:%s\tvalid:%s\ttest:%s\n' % (offline_train_score, offline_valid_score, offline_test_score)) score_file.close() # save prediction results offline_valid_preds_fp = '%s/se_valid.%s.pred' % (self.config.get('DIRECTORY', 'pred_pt'), self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_valid_preds_fp, offline_valid_preds, 'w') offline_test_preds_fp = '%s/se_test.%s.pred' % (self.config.get('DIRECTORY', 'pred_pt'), self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_test_preds_fp, offline_test_preds, 'w')
def run_offline(self): LogUtil.log('INFO', 'cv_tag(%s)' % self.cv_tag) # load feature matrix offline_features = Feature.load_all(self.config.get('DIRECTORY', 'feature_pt'), self.config.get('FEATURE', 'feature_selected').split(), self.config.get('MODEL', 'offline_rawset_name'), self.config.get('FEATURE', 'will_save')) # load labels offline_labels = DataUtil.load_vector('%s/%s.label' % (self.config.get('DIRECTORY', 'label_pt'), self.config.get('MODEL', 'offline_rawset_name')), True) # generate index file if '' == self.cv_tag: self.cv_tag = self.out_tag self.__generate_index(offline_features.shape[0]) # cross validation offline_valid_preds_all = [0.] * offline_features.shape[0] offline_test_preds_all = [0.] * offline_features.shape[0] for fold_id in range(self.cv_num): LogUtil.log('INFO', 'cross validation fold_id(%d) begin' % fold_id) # generate training data set offline_train_pos_rate = float(self.config.get('MODEL', 'train_pos_rate')) offline_train_indexs_fp = '%s/cv_tag%s_n%d_f%d_train.%s.index' % (self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_train_indexs = DataUtil.load_vector(offline_train_indexs_fp, 'int') offline_train_features, offline_train_labels, offline_train_balanced_indexs = \ CrossValidation.__generate_data(offline_train_indexs, offline_labels, offline_features, offline_train_pos_rate) LogUtil.log('INFO', 'offline train data generation done') # generate validation data set offline_valid_pos_rate = float(self.config.get('MODEL', 'valid_pos_rate')) offline_valid_indexs_fp = '%s/cv_tag%s_n%d_f%d_valid.%s.index' % (self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_valid_indexs = DataUtil.load_vector(offline_valid_indexs_fp, 'int') offline_valid_features, offline_valid_labels, offline_valid_balanced_indexs = \ CrossValidation.__generate_data(offline_valid_indexs, offline_labels, offline_features, offline_valid_pos_rate) LogUtil.log('INFO', 'offline valid data generation done') # generate test data set offline_test_pos_rate = float(self.config.get('MODEL', 'test_pos_rate')) offline_test_indexs_fp = '%s/cv_tag%s_n%d_f%d_test.%s.index' % (self.config.get('DIRECTORY', 'index_pt'), self.cv_tag, self.cv_num, fold_id, self.config.get('MODEL', 'offline_rawset_name')) offline_test_indexs = DataUtil.load_vector(offline_test_indexs_fp, 'int') offline_test_features, offline_test_labels, offline_test_balanced_indexs = \ CrossValidation.__generate_data(offline_test_indexs, offline_labels, offline_features, offline_test_pos_rate) LogUtil.log('INFO', 'offline test data generation done') model = Model.new(self.config.get('MODEL', 'model_name'), self.config) model_fp = self.config.get('DIRECTORY', 'model_pt') + '/cv_n%d_f%d.%s.model' % \ (self.cv_num, fold_id, self.config.get('MODEL', 'model_name')) model.save(model_fp) offline_train_preds, offline_valid_preds, offline_test_preds = model.fit(offline_train_features, offline_train_labels, offline_valid_features, offline_valid_labels, offline_test_features, offline_test_labels) offline_train_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_train_labels, offline_train_preds) offline_valid_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_valid_labels, offline_valid_preds) offline_test_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_test_labels, offline_test_preds) score_fp = '%s/%s.score' % (self.config.get('DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write('fold:%d\ttrain:%s\tvalid:%s\ttest:%s\n' % (fold_id, offline_train_score, offline_valid_score, offline_test_score)) score_file.close() # merge prediction results for index in range(len(offline_valid_balanced_indexs)): offline_valid_preds_all[offline_valid_balanced_indexs[index]] = offline_valid_preds[index] for index in range(len(offline_test_balanced_indexs)): offline_test_preds_all[offline_test_balanced_indexs[index]] = offline_test_preds[index] LogUtil.log('INFO', 'cross test fold_id(%d) done' % fold_id) # save prediction results offline_valid_preds_all_fp = '%s/cv_n%d_valid.%s.pred' % (self.config.get('DIRECTORY', 'pred_pt'), self.cv_num, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_valid_preds_all_fp, offline_valid_preds_all, 'w') offline_test_preds_all_fp = '%s/cv_n%d_test.%s.pred' % (self.config.get('DIRECTORY', 'pred_pt'), self.cv_num, self.config.get('MODEL', 'offline_rawset_name')) DataUtil.save_vector(offline_test_preds_all_fp, offline_test_preds_all, 'w') # evaluate offline_valid_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_labels, offline_valid_preds_all) offline_test_score = Evaluator.evaluate(self.config.get('MODEL', 'evaluator_name'), offline_labels, offline_test_preds_all) score_fp = '%s/%s.score' % (self.config.get('DIRECTORY', 'score_pt'), 'cv') score_file = open(score_fp, 'a') score_file.write('cross_validation\tvalid:%s\ttest:%s\n' % (offline_valid_score, offline_test_score)) score_file.close()
def generate_pagerank(G, alpha, max_iter): pr = nx.pagerank(G, alpha=alpha, max_iter=max_iter) LogUtil.log('INFO', 'Graph cal pagerank done') return pr
def generate_graph(config, weight_feature_name, weight_feature_id, reverse): q2id = {} e2weight = {} G = nx.Graph() train_wfs_fs = None test_wfs_fs = None if weight_feature_name is not None: train_wfs_fs = Feature.load( '%s/%s.train.smat' % (config.get('DIRECTORY', 'feature_question_pair_pt'), weight_feature_name)).toarray() test_wfs_fs = Feature.load( '%s/%s.test.smat' % (config.get('DIRECTORY', 'feature_question_pair_pt'), weight_feature_name)).toarray() if 'True' == reverse: LogUtil.log('INFO', 'will reverse') for index in range(len(train_wfs_fs)): train_wfs_fs[index][weight_feature_id] = 1. - train_wfs_fs[ index][weight_feature_id] for index in range(len(test_wfs_fs)): test_wfs_fs[index][weight_feature_id] = 1. - test_wfs_fs[ index][weight_feature_id] fin = csv.reader( open('%s/train.csv' % config.get('DIRECTORY', 'origin_pt'))) fin.next() index = 0 for p in fin: q1 = str(p[3]).strip() q2 = str(p[4]).strip() weight = 0 if train_wfs_fs is None else train_wfs_fs[index][ weight_feature_id] if q1 not in q2id: q2id[q1] = len(q2id) if q2 not in q2id: q2id[q2] = len(q2id) G.add_edge(q2id[q1], q2id[q2], weight=weight) e2weight[(q2id[q1], q2id[q2])] = weight e2weight[(q2id[q2], q2id[q1])] = weight index += 1 fin = csv.reader( open('%s/test.csv' % config.get('DIRECTORY', 'origin_pt'))) fin.next() index = 0 for p in fin: q1 = str(p[1]).strip() q2 = str(p[2]).strip() weight = 0 if test_wfs_fs is None else test_wfs_fs[index][ weight_feature_id] if q1 not in q2id: q2id[q1] = len(q2id) if q2 not in q2id: q2id[q2] = len(q2id) G.add_edge(q2id[q1], q2id[q2], weight=weight) e2weight[(q2id[q1], q2id[q2])] = weight e2weight[(q2id[q2], q2id[q1])] = weight index += 1 LogUtil.log('INFO', 'Graph constructed.') return q2id, e2weight, G
def generate_hits(G, max_iter): hits_h, hits_a = nx.hits(G, max_iter=max_iter) LogUtil.log('INFO', 'Graph cal hits done') return hits_h, hits_a