def get_sparse_matrix(train=None, test=None, params=None, remove_numbers_function=True, debug=True, save=False, load=True, data_dir="data"): """ Get sparse matrix form of the train and test set Parameters ------------------------- Each input is numpy array: train, test, params: See the documentation of tf_idf function save: To save the train and test sprse matrices on . npz format load: To load the train and test sprse matrices from your local machine data_dir: Specify the ro specifi the data directory where the matrices are saved Returns: -------------------------- train: train set in sparce marix form test: test set in sparce matrix form Example ------- >>> train = pd.read_csv("data/train.csv") >>> test = pd.read_csv("data/test.csv") >>> # to create and save the train and test set >>> train_sparse, test_sparse = get_sparse_matrix(train, test, params=None, remove_numbers_function=True, debug=True, save=True, load=False) >>> # to load the sparse matrices from your local machine >>> train, test = get_sparse_matrix(load=True) """ base_dir = data_dir + '/output/' if not os.path.exists(base_dir): os.makedirs(base_dir) name_train = base_dir + 'sparce_train.npz' name_test = base_dir + 'sparce_test.npz' if load: if os.path.exists(name_train) and os.path.exists(name_test): train, test = load_sparse_csr(name_train), load_sparse_csr(name_test) else: raise ValueError("You asked to load the features but they were not found" + "at the specified location: \n{}\n{}".format(name_train, name_test)) else: print('Computing the sparse matrixes, this will take a while...!') train, test = tf_idf(train, test, params, remove_numbers_function, debug) if save: print('Saving train file as {}'.format(name_train)) save_sparse_csr(name_train, train) print('Saving test file as {}'.format(name_test)) save_sparse_csr(name_test, test) return train, test
def calc_weighted_modularity_langs(A_filename, indice2word_filename, langs): """ :param A_filename: :param indice2word_filename: :param langs: prefix tags for each word in the .txt embedding file :return: normalized modularity """ A = load_sparse_csr(A_filename).todok() indice2word = pickle.load(open(indice2word_filename, "rb")) num_community = len(langs) a_l = [0] * num_community e_ll = [0] * num_community e_ll_Q_max = [0] * num_community lang_to_indice = {lang: i for i, lang in enumerate(langs) } # assigns a community ID for each language degree_list, two_m = get_degrees(A) """ a_l computation """ for node_id in range(A.shape[0]): node_word = indice2word[node_id] community = lang_to_indice[node_word[:3]] k_i = degree_list[node_id] a_l[community] += k_i """ e_ll computation """ for processed, e in enumerate(A.items()): node1, node2 = e[0] node1_word = indice2word[node1] node2_word = indice2word[node2] community_id = lang_to_indice[node1_word[:3]] e_ll_Q_max[community_id] += 1 # 1 is maximum possible weight if node1_word[:3] == node2_word[:3]: e_ll[community_id] += A[node1, node2] check_e_ii_s_Q_max = 0 for i in range(len(a_l)): check_e_ii_s_Q_max += e_ll_Q_max[i] assert (check_e_ii_s_Q_max == two_m) e_ll = list(map(lambda x: x / two_m, e_ll)) e_ll_Q_max = list(map(lambda x: x / two_m, e_ll_Q_max)) a_l = list(map(lambda x: x / two_m, a_l)) print("mean degree=%.4f" % (np.mean(degree_list))) Q = 0 Q_max = 0 for i in range(len(a_l)): Q += e_ll[i] - (a_l[i]**2) Q_max += e_ll_Q_max[i] - (a_l[i]**2) print("Weighted Q=%.3f" % Q) print("Weighted Q_max=%.3f" % Q_max) print("Normalized Q=%.3f" % (Q / Q_max)) return Q / Q_max
def __init__(self, db_path=DB_PATH, mat_path=MAT_PATH): self.db_path = db_path self.mat_path = mat_path self.db = DocDB(db_path=self.db_path) self.mat, metadata = utils.load_sparse_csr(self.mat_path) # doc_freqs, hash_size, ngram, doc_dict for k, v in metadata.items(): setattr(self, k, v)
def __init__(self, strict=True, num_workers=8, tokenize=False, topn=100): """ Args: tfidf_path: path to saved model file strict: fail on empty queries or continue (and return empty result) """ # Load from disk self.topn = topn self.tokenize = tokenize self.num_workers = num_workers tfidf_path = DATA_DIR + "/corpus-tfidf-ngram=1-hash=16777216-tokenizer=spacy.npz" logger.info('Loading %s' % tfidf_path) matrix, metadata = utils.load_sparse_csr(tfidf_path) self.doc_mat = matrix self.ngrams = metadata['ngram'] self.hash_size = metadata['hash_size'] self.tokenizer = tokenizers.get_class(metadata['tokenizer'])() self.doc_freqs = metadata['doc_freqs'].squeeze() self.doc_dict = metadata['doc_dict'] self.num_docs = len(self.doc_dict[0]) self.strict = strict
def initCosSimQuery(): global term2id global invertedIndex global W global docCount global termCount global idf term2tidFile = open("term2tid.json", "r") indexFile = open("invertedIndex.json", "r") term2id = json.load(term2tidFile) invertedIndex = json.load(indexFile) term2tidFile.close() indexFile.close() W = utils.load_sparse_csr("weightMatrix.npz") idf = np.load('idf.npy') docCount = np.size(W, 1) termCount = len(term2id)
def initCosSimQuery(): global term2id global invertedIndex global W global docCount global termCount global idf term2tidFile = open("term2tid.json", "r") indexFile = open("invertedIndex.json", "r") term2id = json.load(term2tidFile) invertedIndex = json.load(indexFile) term2tidFile.close() indexFile.close() W = utils.load_sparse_csr("weightMatrix.npz") idf = np.load('idf.npy') docCount = np.size(W, 1) termCount = len(term2id)
def _load_arrays(self, fn): zero_fn = fn + '.npz' bin_fn = fn + '.bin.npz' self.zero_sparse = load_sparse_csr(zero_fn) self.binary_sparse = load_sparse_csr(bin_fn)
print( 'Loading the paragraph trained classifier trained on data processed by ' 'biggest gap thresholding mechanism ') classifier = load_pickle(config.classifier_par_biggest_gap) threshold = 0.91 y_true = process_y(data, threshold_biggest_gap) else: print( 'Loading the paragraph trained classifier trained on data processed by' ' threshold_half_max function') classifier = load_pickle(config.classifier_par_half_max) threshold = 0.39 y_true = process_y(data, threshold_half_max) print("Loading x") x = load_sparse_csr(data['x']) else: threshold = 0.3 vectorizer = load_pickle(config.vectorizer) binarizer = load_pickle(config.binarizer) print('Loading the classifier') classifier = load_pickle(config.classifier) corpus, topics = build_corpus_and_topics(config.data['test']) print('Transforming corpus by vectorizer') x = vectorizer.transform(corpus) print('Transforming article topics by binarizer') y_true = binarizer.transform(topics)
def load_active_during_test(): items_data = pd.read_csv("data_modified/item_profile.csv", delim_whitespace=True, dtype={'id': int, 'active_during_test': str}, usecols=['id', 'active_during_test']) return set(items_data[items_data.active_during_test == '1']['id'].values) active_items = load_active_during_test() # load data from csv interactions_map = load_interactions() urm_user = load_sparse_csr('urm_user_based_full.npz') urm_item = load_sparse_csr('urm_item_based_full.npz') urm_funk = load_sparse_csr('urm_funk_full.npz') urm_user = normalize(urm_user) urm_item = normalize(urm_item) urm_funk = normalize(urm_funk) # 0.024365000000000008 0.12 0.24 alpha = 0.3 beta = 0.3 # for beta in np.arange(start=0.28, stop=10, step=0.01): estimated_urm = urm_item * alpha + urm_user * beta + urm_funk * (1.0 - alpha - beta) # write recommendations
parser = argparse.ArgumentParser() parser.add_argument('ct_path', type=str, default=None, help='Path to count matrices') parser.add_argument('out_dir', type=str, default=None, help='Directory for saving output files') args = parser.parse_args() ct_files = [f for f in utils.iter_files(args.ct_path)] logger.info('Loading the zeroth count matrix...') mat, metadata = utils.load_sparse_csr(ct_files[0]) DOC2IDX, doc_ids = metadata['doc_dict'] for i in range(1, len(ct_files)): logger.info('Loading %ith count matrix...' % i) nxt_mat, nxt_metadata = utils.load_sparse_csr(ct_files[i]) if metadata['hash_size'] != nxt_metadata['hash_size']: raise RuntimeError('hash_size not equal in %ith file' % i) if metadata['ngram'] != nxt_metadata['ngram']: raise RuntimeError('ngram not equal in %ith file' % i) logger.info('Merging...') mat = sp.hstack([mat, nxt_mat])
default=None, help='Path to count matrix file') parser.add_argument('out_dir', type=str, default=None, help='Directory for saving output files') parser.add_argument('--model', type=str, default='tfidf', help=('tfidf or pmi')) args = parser.parse_args() logger.info('Loading count matrix...') count_matrix, metadata = utils.load_sparse_csr(args.ct_path) logger.info('Making %s vectors...' % args.model) if args.model == 'tfidf': mat = get_tfidf_matrix(count_matrix) elif args.model == 'pmi': mat = get_pmi_matrix(count_matrix) else: raise RuntimeError('Model %s is invalid' % args.model) basename = os.path.splitext(os.path.basename(args.ct_path))[0] basename = ('%s-' % args.model) + basename if not os.path.exists(args.out_dir): logger.info("Creating data directory")
import utils import numpy as np from scipy import sparse from sklearn.externals import joblib #root to this code is: /Google Drive/CIKM_AnalytiCup_2017/Code/CIKM_AnalytiCup_2017/ filename = 'features/features_title_desc_attributes_1_3_word_1_5_char.npz' X = utils.load_sparse_csr(filename) XX = utils.load_sparse_csr(filename.split('.')[-2] + '_valid.npz') loaded = np.load(filename.split('.')[-2] + '_dict.npz') words = loaded['words'].tolist() chars = loaded['chars'].tolist() feature_names = [ 'title.getLengths', 'title.getLengthsByTerm', 'title.getSpecialCharStat', 'title.hasSpecialChar', 'title.getUpperCharStat', 'title.getNounStat', 'title.getVerbStat', 'title.getAdjectiveStat', 'title.hasNumber', 'title.getNumberStat', 'title.hasTamilChar', 'title.hasChineseChar', 'title.getNonEnglishCharStat', 'title.getColorStat', 'title.getBrandStat(brands)', 'title.getSyllableStat', 'title.getPolysyllabStat',