def calculate_optimal_k(self, body: str, algorithm: str = 'kmeans', min_length: int = 40, max_length: int = 600, k_max: int = None): """ Calculates the optimal Elbow K. :param body: The input body to summarize. :param algorithm: The algorithm to use for clustering. :param min_length: The min length to use. :param max_length: The max length to use. :param k_max: The maximum number of clusters to search. :return: """ sentences = self.sentence_handler(body, min_length, max_length) if k_max is None: k_max = len(sentences) - 1 hidden = self.model(sentences, self.hidden, self.reduce_option, hidden_concat=self.hidden_concat) optimal_k = ClusterFeatures( hidden, algorithm, random_state=self.random_state).calculate_optimal_cluster(k_max) return optimal_k
def calculate_elbow( self, body: str, algorithm: str = 'kmeans', min_length: int = 40, max_length: int = 600, k_max: int = None, ) -> List[float]: """ Calculates elbow across the clusters. :param body: The input body to summarize. :param algorithm: The algorithm to use for clustering. :param min_length: The min length to use. :param max_length: The max length to use. :param k_max: The maximum number of clusters to search. :return: List of elbow inertia values. """ sentences = self.sentence_handler(body, min_length, max_length) if k_max is None: k_max = len(sentences) - 1 hidden = self.model(sentences, self.hidden, self.reduce_option, hidden_concat=self.hidden_concat) elbow = ClusterFeatures( hidden, algorithm, random_state=self.random_state).calculate_elbow(k_max) return elbow
def run_clusters(self, content: List[str], ratio=0.2, algorithm='kmeans', use_first: bool= True) -> List[str]: hidden = self.model(content, self.hidden, self.reduce_option) hidden_args = ClusterFeatures(hidden, algorithm, random_state=self.random_state).cluster(ratio) if use_first: if hidden_args[0] != 0: hidden_args.insert(0,0) return [content[j] for j in hidden_args]
def cluster_runner( self, content: List[str], ratio: float = 0.2, algorithm: str = 'kmeans', use_first: bool = True, num_sentences: int = None) -> Tuple[List[str], np.ndarray]: """ Runs the cluster algorithm based on the hidden state. Returns both the embeddings and sentences. :param content: Content list of sentences. :param ratio: The ratio to use for clustering. :param algorithm: Type of algorithm to use for clustering. :param use_first: Whether to use first sentence (helpful for news stories, etc). :param num_sentences: Number of sentences to use for summarization. :return: A tuple of summarized sentences and embeddings """ if num_sentences is not None: num_sentences = num_sentences if use_first else num_sentences #hidden = self.model(content, self.hidden, self.reduce_option) my_path = os.path.abspath(os.path.dirname(__file__)) model_path = os.path.join( *[my_path, "distilroberta-base-paraphrase-v1"]) print("path====", model_path) #self.model = SentenceTransformer(model_name_or_path=model_path) self.model = SentenceTransformer("distilroberta-base-paraphrase-v1") hidden = self.model.encode(content, convert_to_tensor=True) hidden_args = ClusterFeatures(hidden, algorithm, random_state=self.random_state).cluster( ratio, num_sentences) if use_first: if not hidden_args: hidden_args.append(0) elif hidden_args[0] != 0: hidden_args.insert(0, 0) sentences = [content[j] for j in hidden_args] #embeddings = np.asarray([hidden[j] for j in hidden_args]) embeddings = hidden return sentences, embeddings
def cluster_runner( self, content: List[str], ratio: float = 0.2, algorithm: str = 'kmeans', use_first: bool = True, num_sentences: int = None ) -> Tuple[List[str], np.ndarray]: """ Runs the cluster algorithm based on the hidden state. Returns both the embeddings and sentences. :param content: Content list of sentences. :param ratio: The ratio to use for clustering. :param algorithm: Type of algorithm to use for clustering. :param use_first: Whether to use first sentence (helpful for news stories, etc). :param num_sentences: Number of sentences to use for summarization. :return: A tuple of summarized sentences and embeddings """ if num_sentences is not None: num_sentences = num_sentences if use_first else num_sentences #NASH #hidden = self.model(content, self.hidden, self.reduce_option) self.model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens') hidden = self.model.encode(content, convert_to_tensor=True) hidden_args = ClusterFeatures(hidden, algorithm, random_state=self.random_state).cluster(ratio, num_sentences) if use_first: if not hidden_args: hidden_args.append(0) elif hidden_args[0] != 0: hidden_args.insert(0, 0) sentences = [content[j] for j in hidden_args] #NASH #embeddings = np.asarray([hidden[j] for j in hidden_args]) embeddings = hidden return sentences, embeddings
def cluster_runner( self, content: List[str], ratio: float = 0.2, algorithm: str = 'kmeans', use_first: bool = True, num_sentences: int = None) -> Tuple[List[str], np.ndarray]: """ Runs the cluster algorithm based on the hidden state. Returns both the embeddings and sentences. :param content: Content list of sentences. :param ratio: The ratio to use for clustering. :param algorithm: Type of algorithm to use for clustering. :param use_first: Return the first sentence in the output (helpful for news stories, etc). :param num_sentences: Number of sentences to use for summarization. :return: A tuple of summarized sentences and embeddings """ if num_sentences is not None: num_sentences = num_sentences if use_first else num_sentences hidden = self.model(content, self.hidden, self.reduce_option, hidden_concat=self.hidden_concat) hidden_args = ClusterFeatures(hidden, algorithm, random_state=self.random_state).cluster( ratio, num_sentences) if use_first: if not hidden_args: hidden_args.append(0) elif hidden_args[0] != 0: hidden_args.insert(0, 0) sentences = [content[j] for j in hidden_args] embeddings = np.asarray([hidden[j] for j in hidden_args]) return sentences, embeddings
def cluster_runner( self, content: List[str], ratio: float = 0.2, algorithm: str = 'kmeans', use_first: bool = True, num_sentences: int = None) -> Tuple[List[str], np.ndarray]: """ Runs the cluster algorithm based on the hidden state. Returns both the embeddings and sentences. :param content: Content list of sentences. :param ratio: The ratio to use for clustering. :param algorithm: Type of algorithm to use for clustering. :param use_first: Whether to use first sentence (helpful for news stories, etc). :param num_sentences: Number of sentences to use for summarization. :return: A tuple of summarized sentences and embeddings """ def find_closest_args(centroids, features): args = {} for i, feature in enumerate(features): dist_list = [] for j, centroid in enumerate(centroids): value = np.linalg.norm(feature - centroid) dist_list.append([value, j]) args[i] = [min(dist_list, key=lambda x: x[0])] return args if num_sentences is not None: num_sentences = num_sentences if use_first else num_sentences hidden = self.model(content, self.hidden, self.reduce_option, hidden_concat=self.hidden_concat) self.features = hidden hidden_args = ClusterFeatures(hidden, algorithm, random_state=self.random_state).cluster( ratio, num_sentences) self.centroids = hidden_args[1] hidden_args = hidden_args[0] self.hidden_args = hidden_args if use_first: if not hidden_args: hidden_args.append(0) elif hidden_args[0] != 0: hidden_args.insert(0, 0) sentences = [content[j] for j in hidden_args] embeddings = np.asarray([hidden[j] for j in hidden_args]) args = find_closest_args(self.centroids, self.features) df = pd.DataFrame([a, b[0][0], b[0][1]] for a, b in args.items()) df.columns = ['Id', 'Weight', 'Centroid'] scaler = MinMaxScaler() df['Weight'] = scaler.fit_transform(df['Weight'].to_numpy().reshape( -1, 1)) df['Weight'] = df['Weight'] - 1 df['Weight'] = df['Weight'].abs().round(4) df['Sent'] = df['Id'].apply(lambda x: content[x]) self.df = df return sentences, embeddings