def graph_communities(self, data, min_value_=0.6, topn_=400, k1=30, expected_density=0.1, graph_communities_df=None): if not graph_communities_df is None: self.graph_clusters = graph_communities_df return graph_communities_df graph_clusters = product_finder(data, min_value_=min_value_, topn_=topn_, k1=k1, expected_density=expected_density) try: graph_clusters = apply_word_embedings( graph_clusters['clustered_data'], model_name=self.model_name ).assign( starting_date=graph_clusters['clustered_data'].date_min, last_modified_date=graph_clusters['clustered_data'].date_max) except: graph_clusters = apply_word_embedings( graph_clusters['clustered_data'], model_name=self.model_name).assign( starting_date=datetime.datetime.today().strftime( '%Y-%m-%d'), last_modified_date=datetime.datetime.today().strftime( '%Y-%m-%d')) if not 'counter' in graph_clusters.columns: self.graph_clusters = graph_clusters.assign(counter=1) else: self.graph_clusters = graph_clusters return self.graph_clusters
def linkage(self, title_clusters, method='ward'): try: data = np.array([i[0][0] for i in title_clusters.word_vector]) Z = fastcluster.linkage_vector(data, method=method) except AttributeError: title_clusters = apply_word_embedings(title_clusters) data = np.array([i[0][0] for i in title_clusters.word_vector]) Z = fastcluster.linkage_vector(data, method=method) return Z
def linkage(self, title_clusters, method='ward', linkage_matrix=None): if not linkage_matrix is None: self.linkage_matrix = linkage_matrix return linkage_matrix try: data = np.array([i[0][0] for i in title_clusters.word_vector]) Z = fastcluster.linkage_vector(data, method=method) except AttributeError: title_clusters = apply_word_embedings(title_clusters, model_name=self.model_name) data = np.array([i[0][0] for i in title_clusters.word_vector]) Z = fastcluster.linkage_vector(data, method=method) return Z
def search_engine_fasttext(data_ , productsDB ,min_sim=0.9,topn = 1,model_name='model_fast_text_sg_40',column_name_db = 'word_vector',column_name_data = 'word_vector',model_folder_path = r'C:\ProductClustering\ProductsDB\fasttext_models\\', metric ='cosine',pre_computed_word_vectors = True): g = os.path.join(os.path.dirname(model_folder_path), model_name) model_fast_text = pickle.load(open(g, 'rb')) print('generating reference csr_matrix from data') reference_cv_matrix = csr_matrix(np.array([i[0][0] for i in productsDB[column_name_db]])) data_ = data_.assign(product_id_fasttext = 0) reference_product_id = productsDB['product_id'] if not pre_computed_word_vectors: print('applying word embedings to data') data = apply_word_embedings(data_) cv_matrix = csr_matrix(np.array([i[0][0] for i in data[column_name_data]])) print('computing similarity matrix. metric = {}'.format(metric)) sim_matrix = pairwise_cosine_sparse_sim(cv_matrix,reference_cv_matrix, topn = topn, min_value=min_sim) iterable = sim_matrix.shape[0] labels = [] if topn > 1: for ad in range(iterable): if np.max(sim_matrix[ad].A) >= min_sim: labels.append(reference_product_id.iloc[np.argmax(sim_matrix[ad].A)]) else: labels.append(-1) print(ad) else: for ad in range(iterable): if np.max(sim_matrix[ad].A) >= min_sim: labels.append(reference_product_id.iloc[sim_matrix[ad].nonzero()[1][0]]) else: labels.append(-1) print(ad) data_ = data.assign(product_id_fasttext = labels) else: cv_matrix = csr_matrix(np.array([i[0][0] for i in data_[column_name_data]])) print('computing similarity matrix. metric = {}'.format(metric)) sim_matrix = pairwise_cosine_sparse_sim(cv_matrix,reference_cv_matrix, topn = topn, min_value=min_sim) iterable = sim_matrix.shape[0] labels = [] for ad in range(iterable): if np.max(sim_matrix[ad].A) > min_sim: labels.append(reference_product_id.iloc[np.argmax(sim_matrix[ad].A)]) else: labels.append(-1) print(ad) data_ = data_.assign(product_id_fasttext = labels) return data_
def get_similar_products(self, product_id=123, top_n=10, title=None, column_name='word_vector'): if type(title) == None: try: csr1 = csr_matrix( np.array([ i[0][0] for i in self.products_db[column_name][ self.products_db['product_id'].isin(product_id)] ])) ads = product_id except: csr1 = csr_matrix( np.array([ i[0][0] for i in self.products_db[column_name][ self.products_db['product_id'].isin([product_id])] ])) ads = [product_id] else: if type(title) == str: title = [title] titles = apply_word_embedings(pd.DataFrame({'ad_title': title}), model_name=self.model_name) csr1 = csr_matrix( np.array([i[0][0] for i in titles['word_vector']])) ads = title csr2 = csr_matrix( np.array([i[0][0] for i in self.products_db[column_name]])) sim_matrix = pairwise_cosine_sparse_sim(csr1, csr2, topn=top_n, min_value=0, word_embedings=True) labels = {} for index, ad in enumerate(ads): labels[ad] = self.products_db.iloc[np.where( sim_matrix[index].A > 0.01)[1]].assign( similarity=sim_matrix[index].A[np.where( sim_matrix[index].A > 0.01)]).sort_values( by='similarity', ascending=False) return labels
def graph_communities(self, sample, min_value_=0.6, topn_=400, k1=30, expected_density=0.1): graph_clusters = product_finder(sample, min_value_=min_value_, topn_=topn_, k1=k1, expected_density=expected_density) graph_clusters = apply_word_embedings( graph_clusters['clustered_data']).assign( starting_date=datetime.datetime.today().strftime('%Y-%m-%d'), last_modified_date=datetime.datetime.today().strftime( '%Y-%m-%d')) self.graph_clusters = graph_clusters.assign(counter=1) return graph_clusters.assign(counter=1)
def handle_unlabeled(self, data, max_product_id, clustering_algorithm='agglomerative'): unknown_products = apply_word_embedings(data, model_name=self.model_name) if clustering_algorithm == 'agglomerative': unknown_data = np.array( [i[0][0] for i in unknown_products.word_vector]) cluster_ = fastcluster.linkage_vector(unknown_data, method='ward') cluster_labels = Cluster.hierarchy.fcluster(cluster_, 0.2) unknown_products = unknown_products.assign( product_id=cluster_labels) elif clustering_algorithm == 'community': unknown_products = self.graph_communities( unknown_products, min_value_=0.8, topn_=400, k1=50, expected_density=0.1, graph_communities_df=None)
a = products_db_finder(model_name=model_name) data = get_data( period=360, final_date_string='today', date_format="%Y-%m-%d", query_path='C:\ProductClustering\sql_queries\categories_metrics.txt') cluster_graph = a.graph_communities( data[['ad_id', 'ad_title', 'category_id', 'date_min', 'date_max']]) g1 = a.group_by_product(cluster_graph) link = a.linkage(g1) g1.word_vector g1 = a.group_by_product( apply_word_embedings( pd.read_csv(r'C:\ProductClustering\productsDB\classtreste.csv'))) link = pickle.load( open(r'C:\ProductClustering\productsDB\model_fast_text_sg_40_linkage', 'rb')) g1 = a.group_by_product(cluster_graph) hierarchy = g1.assign( product_id=a.hierarchycal_clustering(link, threshold=0.3)) hierarchy[hierarchy.counter > 2].product_id.nunique() g2 = a.group_by_product(hierarchy) g3 = g2[g2.counter > 2] a.export_db_dic('C:\ProductClustering\productsDB\products_db_objects\seeds\\', seed_name_export) teste = a.get_similar_products(product_id=123,