def result(self): data_df = self.clean_data.data() all_data_df = self.clean_data.getSpambase_data() target_df = self.clean_data.target() # Defining Model model = TSNE(learning_rate=100) # Fitting Model transformed = model.fit_transform(all_data_df) # Plotting 2d t-Sne x_axis = transformed[:, 0] pprint.pprint(x_axis) y_axis = transformed[:, 1] pprint.pprint(y_axis) plt.scatter(x_axis, y_axis, c=target_df) #plt.show() plt.savefig(self.file_png) # Create the visualizer and draw the vectors tfidf = TfidfVectorizer() docs = tfidf.fit_transform(data_df) tsne = TSNEVisualizer() tsne.fit(docs, target_df) tsne.poof()
def tsne_visualization(dataset_object, num_examples): """ Produce and save T-SNE visualization of feature vectors for given dataset Parameters ---------- dataset_object: tv_net.dataset.Dataset dataset object containing feature vectors and class names num_examples: int number of examples to plot """ dataset_object.shuffle_examples( ) # shuffle so that we don't get all one class feature_vectors = np.array( [item.feature_vector for item in dataset_object.items[:num_examples]]) label_list = [ item.class_name for item in dataset_object.items[:num_examples] ] title = 'T-SNE of feature vectors extracted from baseline classifier - using random sample of {} images'.format( num_examples) tsne = TSNEVisualizer(colormap='rainbow', title=title) tsne.fit(feature_vectors, label_list) output_path = os.path.join(dataset_object.config.OUTPUT_DIR, 'visualizations', 'feature_vector_tsne.png') tsne.show(outpath=output_path) tsne.show() # have to repeat to show and save
def tsne_kmeans_clusters(tfidf, num_clusters=[3, 5, 7, 9, 11]): ''' Vectorizer results are normalized, which makes KMeans behave as spherical k-means for better results. Since LSA/SVD results are not normalized, we have to redo the normalization. ''' print( '\nUse sklearn tSNE to visualize viability of cluster estimates to inform n topic choices: {}' .format(num_clusters)) for k in num_clusters: start = datetime.now() svd = TruncatedSVD(n_components=50, n_iter=10, random_state=0) normalizer = Normalizer(copy=False) lsa = make_pipeline(svd, normalizer) reduced = lsa.fit_transform(tfidf) # next, apply kmeans to the corpus to get labels clusters = KMeans(n_clusters=k, init='k-means++') clusters.fit(reduced) tsne = TSNEVisualizer(decompose=None) tsne.fit(reduced, ["cluster {}".format(c) for c in clusters.labels_]) tsne.finalize() filename = r'images/tsne_projections/tSNE_wKMeans_SVD_' + str( k) + '_clusters_' + str(tfidf.shape[0]) + '_docs.png' plt.savefig(filename) plt.close() end = datetime.now() print(' ' + filename) print(" Time taken: {}".format(end - start))
def text_cluster_tsne(text_vector, TextVectorizer=TfidfVectorizer, text_kwargs=text_kwargs, n_clusters=10, labels=None): '''Uses a TextVectorizer to transform the text contained (at the sentence or paragraph level) in the text_vector arg to produce a TSNE visualization. The label for the final plot is clusters produced from KMeans if labels are not passed. ARGS: text_vector <np.array>: Vector of text units. Must be type str. KWARGS: TextVectorizer <sklearn.feature_extraction.text>: Transformer. text_kwargs <dict>: kwargs to pass to TextVectorizer n_clusters <int>: If not using labels, number of clusters in KMeans labels <np.array>: True categorical labels. Discrete. RETURNS: None, prints visualizations to the console. ''' txt_vctzr = TextVectorizer(**text_kwargs) docs = txt_vctzr.fit_transform(text_vector) tsne = TSNEVisualizer() if labels is None: # derive clusters if labels not provided clusters = KMeans(n_clusters=n_clusters) clusters.fit(docs) tsne.fit(docs, ["cluster_{}".format(c) for c in clusters.labels_]) else: # otherwise use labels tsne.fit(docs, labels) sns.despine() tsne.poof()
def visualize_yellowbrick(dim_reduction, encoding, corpus_data, corpus_target, labels=True, alpha=0.7, metric=None): # https://pypi.org/project/yellowbrick/ # https://github.com/DistrictDataLabs/yellowbrick/tree/develop/examples # https://medium.com/@sangarshananveera/rapid-text-visualization-with-yellowbrick-51d3499c9333 if 'tfidf' in encoding.lower(): encode = TfidfVectorizer() if 'count' in encoding.lower(): encode = CountVectorizer() docs = encode.fit_transform(corpus_data) if labels is True: labels = corpus_target else: labels = None if 'umap' in dim_reduction.lower(): if metric is None: viz = UMAPVisualizer() else: viz = UMAPVisualizer(metric=metric) if 't-sne' in dim_reduction.lower(): viz = TSNEVisualizer(alpha=alpha) viz.fit(docs, labels) return viz.poof()
def tsne(): corpus = load_hobbies() docs = TfidfVectorizer().fit_transform(corpus.data) oz = TSNEVisualizer(ax=newfig()) oz.fit(docs, corpus.target) savefig(oz, "corpus_tsne")
def perform_tsne(X, Y, vec=None, outpath="", clusterLabels=False, savePlot=False): if vec == None: vec = TfidfVectorizer(preprocessor=identity, tokenizer=identity) docs = vec.fit_transform(X) labels = Y # from yellowbrick.text import TSNEVisualizer tsne = TSNEVisualizer() if clusterLabels: tsne.fit(docs, ["c{}".format(c) for c in Y]) # where Y=clusters.labels_ else: tsne.fit(docs, labels) if savePlot: # tsne.finalize() tsne.poof(outpath=outpath) else: tsne.poof()
def clusters_tsne(self, labels: pd.Series, title: str = 'title'): tsne = TSNEVisualizer(random_state=42) tsne.fit(self.vectors, labels) f = tsne.show().figure f.set_figheight(15) f.set_figwidth(15) f.suptitle(title) return f
def tsne_pack(c, l): my_title = "t-SNE Plot of " + c + " feature" data = df.filter(like=c) tfidf = TfidfVectorizer() new_values = tfidf.fit_transform(corpus) tsne = TSNEVisualizer(title=my_title) tsne.fit(data, l) tsne.poof()
def tsne(docs, target, outpath, **kwargs): # Create a new figure and axes fig = plt.figure() ax = fig.add_subplot(111) # Visualize the frequency distribution visualizer = TSNEVisualizer(ax=ax, **kwargs) visualizer.fit(docs, target) visualizer.poof(outpath=outpath)
def analyse_2_step_model(): X_test = np.load( "test_prepared.npy").item() # this is our Single point of truth #test_silhouette(30, X_test) test = X_test[0:1000] prediction = test_entire_model()[0:1000] vis_shilouette(test, prediction) plt.savefig("silhouette.png") tsne = TSNEVisualizer(colormap=cm.get_cmap('jet', len(set(prediction)))) tsne.fit(test[0:1000], ["c{}".format(c) for c in prediction]) tsne.poof(outpath="tsne.png")
def tsne(c, l): my_title = "t-SNE Plot of final model" data = c tfidf = TfidfVectorizer() new_values = tfidf.fit_transform(corpus) tsne = TSNEVisualizer(title=my_title) tsne.fit(data, l) tsne.poof() # %%time figure(figsize=(20, 10)) tsne(final, label_bias3) # %%time figure(figsize=(20, 10)) tsne(final, label_fact)
def plot_tsne_clusters(corpus, fileids=None, labels=None): from yellowbrick.text import TSNEVisualizer from sklearn.feature_extraction.text import TfidfVectorizer words = corpus.title_tagged(fileids=fileids) normalizer = Corpus_Vectorizer.TextNormalizer() normed = (sent for title in normalizer.transform(words) for sent in title) # normed = (dd for dd in normalizer.transform(docs)) tfidf = TfidfVectorizer() procd = tfidf.fit_transform(normed) tsne = TSNEVisualizer() if labels is None: tsne.fit(procd) else: tsne.fit(procd, ["c{}".format(c) for c in labels]) tsne.poof()
def generate_tsne(title, X, labels): fig, (ax1) = plt.subplots(1, 1, figsize=(4, 2)) title_dic = {'fontsize': 7, 'fontweight': 'bold'} colors = resolve_colors(11, 'Spectral_r') colors2 = resolve_colors(10, 'BrBG_r') tsne = TSNEVisualizer(ax1, colors=colors + colors2,decompose=None) tsne.fit(X, labels) tsne.finalize() ax1 = tsne.ax ax1.set_title(title, title_dic) path = os.path.join(OUTPUT) filename = title filename = os.path.join(path, filename) plt.savefig(filename)
def tsne(ax, classes=True): from sklearn.feature_extraction.text import TfidfVectorizer from yellowbrick.text import TSNEVisualizer X, y = load_data("hobbies", text=True) if not classes: y = None freq = TfidfVectorizer(input='filename', stop_words='english') X = freq.fit_transform(X) visualizer = TSNEVisualizer(ax=ax) visualizer.title = "t-SNE Projection of the Hobbies Corpus" if not classes: visualizer.title = "Unlabeled " + visualizer.title visualizer.fit(X, y) return visualizer
def cluster(corpus, k): y = [i[0] for i in corpus] corpus = [i[1] for i in corpus] eng = list(set(stopwords.words('english'))) trump = [ 'wall', 'president', 'trump', 'loss', 'yes', 'sorry', 'mr', 'build', 'thank', 'people' ] s_w = eng + trump vectorizer = TfidfVectorizer(stop_words=s_w) vectorizer.fit(corpus) features = vectorizer.transform(corpus) tsne = TSNEVisualizer() tsne.fit(features, y) tsne.show()
def update_justifications_data(text, assignment_name, rating_type): if(assignment_name is None): return None global db db.close() db.connect() if(text=='Decision Point'): all_justifications, title, custom_stopwords = get_dp_justifications(assignment_name) elif(text=='Innovation Ratings'): if(rating_type is None): return None assignment = assignment_name.split()[0] pre_post = assignment_name.split()[1].split('-')[0] all_justifications, title, custom_stopwords = get_ratings_justifications(assignment, pre_post, rating_type) custom_stopwords.extend(['wa', 'could', 'also', 'would', 'ha', 'i', 'p', 'g', 'this', 'the']) all_justifications = all_justifications[ [True if len(x)>2 else False for x in all_justifications['justification']] ] all_justifications['justification'].replace('', np.nan, inplace=True) all_justifications.dropna(inplace=True) all_justifications['original'] = all_justifications['justification'] all_justifications['justification'] = all_justifications['justification'].apply(lambda x: lemmatize(x)) all_justifications['sentiment'] = all_justifications['justification'].apply(lambda x: calcParagraphSentiment(x)) tfidf = TfidfVectorizer(stop_words=custom_stopwords) docs = tfidf.fit_transform(list(all_justifications['justification'])) tsne = TSNEVisualizer(random_state=14) transformer = tsne.make_transformer() data = transformer.fit_transform(docs) all_justifications['x'] = data[:, 0] all_justifications['y'] = data[:, 1] return {'data': all_justifications.to_dict(), 'title': title, 'custom_stopwords': custom_stopwords}
def tsne_plot(self, outpath, sample_size=1000, tfidf=True): """ Creates a png file at `outpath` with t-SNE visualization. `sample_size` determines the size of the random sample from each label. Uses TfidfVectorizer by default; if `tfidf` is set to False, CountVectorizer is used. ----------------------------------------------------------------------- More info: https://www.scikit-yb.org/en/latest/api/text/tsne.html https://lvdmaaten.github.io/tsne/ """ if self.tokenizer is None: print('No tokenizer was loaded.') return None df = pd.DataFrame(columns=self.data.columns) for label in self.labels: samp_df = self.data \ .query("Label == @label") \ .sample(sample_size, random_state=19) df = df.append(samp_df, ignore_index=True) # vectorize if tfidf: vectorizer = TfidfVectorizer(tokenizer=self.tokenizer.tokenize) else: vectorizer = CountVectorizer(tokenizer=self.tokenizer.tokenize) X = vectorizer.fit_transform(df.Text) y = df.Label # create the visualizer and draw the vectors tsne = TSNEVisualizer() tsne.fit(X, y) tsne.show(outpath=outpath) return None
def analyse_results(): rerun = False if ("rerun" in sys.argv): print("Redo everything") rerun = True X_test = np.load("test_prepared.npy").item() results = [] names = [] for filename in os.listdir("results"): if filename.endswith(".npy"): if filename[:-4] + "tsne.png" in os.listdir( "results") and not rerun: continue results.append(np.load("results/" + filename)) names.append(filename[:-4]) for i in range(len(results)): print("iteration " + str(i + 1) + " of " + str(len(results)) + " : " + names[i]) vis_shilouette(X_test, results[i]) plt.savefig("results/" + names[i] + "silhouette.png") plt.close() plt.figure() tsne = TSNEVisualizer(colormap=cm.get_cmap( 'jet', len(set(results[i][0:5000]))), alpha=0.5, random_state=45) # make it deterministic tsne.fit(X_test[0:5000], ["c{}".format(c) for c in results[i][0:5000]]) tsne.poof(outpath="results/" + names[i] + "tsne.png", clear_figure=True)
def plotData(base, labels=[-1,0,1]): vectorizer = CountVectorizer(lowercase=False) tweets = vectorizer.fit_transform(base['Tweet']) tsne = TSNEVisualizer() tsne.fit(tweets, labels) tsne.poof()
from scdv import SCDVVectorizer from yellowbrick.datasets import load_hobbies from yellowbrick.text import TSNEVisualizer # from sklearn.feature_extraction.text import TfidfVectorizer if __name__ == "__main__": corpus = load_hobbies() vectorizer = SCDVVectorizer( # w2v_model_path="word2vec.gensim.model", size=400, n_components=10, iter=200, sparsity_percentage=0.1) X = vectorizer.fit_transform(corpus.data) y = corpus.target # vectorizer.save("./sample_vectorizer.pkl") # loaded_vectorizer = SCDVVectorizer.load("./sample_vectorizer.pkl") # X = loaded_vectorizer.transform(corpus) # Create the visualizer and draw the vectors tsne = TSNEVisualizer(decompose_by=20, random_state=42, perplexity=15, learning_rate=200.0, n_iter=10000) tsne.fit(X, y) tsne.show()
def main(X_train_smart, X_test_smart, y_train_smart, y_test_smart, X_train_bank, X_test_bank, y_train_bank, y_test_bank, args): # em = KMeans(n_clusters=4, random_state=27) # em.fit(X_train_smart) # prediction = em.predict(X_train_smart) # viz = RadViz() # viz.fit_transform(X_train_smart, prediction) # viz.show() # umap = UMAPVisualizer() # umap.fit(X_train_smart, ["c{}".format(c) for c in prediction]) # umap.show() # tsne = TSNEVisualizer(decompose_by=4) # tsne.fit(X_train_smart, ["c{}".format(c) for c in prediction]) # tsne.show() # exit() sil_score_list_smart = [] cal_har_score_list_smart = [] davies_bouldin_score_list_smart = [] sil_score_list_bank = [] cal_har_score_list_bank = [] davies_bouldin_score_list_bank = [] num_clusters_list = np.arange(2, 25) for num_clusters in num_clusters_list: k_means = KMeans(n_clusters=num_clusters, random_state=27) k_means.fit(X_train_smart) prediction = k_means.predict(X_train_smart) # print(prediction) sil_score_list_smart.append(silhouette_score(X_train_smart, prediction)) cal_har_score_list_smart.append( calinski_harabasz_score(X_train_smart, prediction)) davies_bouldin_score_list_smart.append( davies_bouldin_score(X_train_smart, prediction)) for num_clusters in num_clusters_list: k_means = KMeans(n_clusters=num_clusters, random_state=27) k_means.fit(X_train_bank) prediction = k_means.predict(X_train_bank) # print(prediction) sil_score_list_bank.append(silhouette_score(X_train_bank, prediction)) cal_har_score_list_bank.append( calinski_harabasz_score(X_train_bank, prediction)) davies_bouldin_score_list_bank.append( davies_bouldin_score(X_train_bank, prediction)) with open('experiment_best.json') as f: params = json.load(f) if args.dimensionality is None: num_clusters_smart = params['k_means']['smart'] num_clusters_bank = params['k_means']['bank'] else: num_clusters_smart = params[args.dimensionality[0]]['k_means']['smart'] num_clusters_bank = params[args.dimensionality[0]]['k_means']['bank'] # Scale these for plotting cal_har_score_list_smart = [x / 500 for x in cal_har_score_list_smart] cal_har_score_list_bank = [x / 500 for x in cal_har_score_list_bank] davies_bouldin_score_list_smart = [ x / 5 for x in davies_bouldin_score_list_smart ] davies_bouldin_score_list_bank = [ x / 5 for x in davies_bouldin_score_list_bank ] plt.rc("font", size=8) plt.rc("axes", titlesize=12) plt.rc("axes", labelsize=10) plt.rc("xtick", labelsize=8) plt.rc("ytick", labelsize=8) plt.rc("legend", fontsize=8) plt.rc("figure", titlesize=11) #fig, ax = plt.subplots(2, 1, dpi=100, sharex=True, figsize=(5,4)) fig, ax = plt.subplots(1, 4, figsize=(15, 4)) fig.suptitle( 'K-Means Clusters - # of clusters Analysis (Left: Smart Grid, Right: Bank Loan)', fontsize=14) # ax[0].plot(problem_size_list, sa_fitness_list, 'b-', label='Simulated Annealing', linewidth=1) # ax[0].plot(problem_size_list, ga_fitness_list, 'g:', label='Genetic', linewidth=1) ax[0].plot(num_clusters_list, sil_score_list_smart, 'b-', label='Silhouette', linewidth=1) ax[0].plot(num_clusters_list, cal_har_score_list_smart, 'r--', label='Calinksi-Harabasz / 500', linewidth=1) ax[0].plot(num_clusters_list, davies_bouldin_score_list_smart, 'g-.', label='Davies-Bouldin / 5', linewidth=1) ax[0].set(xlabel='K (# of clusters)', ylabel='Scores') ax[0].set_title('Clustering Scores') ax[0].legend() k_means = KMeans(n_clusters=num_clusters_smart, random_state=27) k_means.fit(X_train_smart) prediction_smart = k_means.predict(X_train_smart) tsne = TSNEVisualizer(decompose_by=X_train_smart.shape[1] - 1, ax=ax[1], random_state=27) tsne.fit(X_train_smart, ["c{}".format(c) for c in prediction_smart]) ax[1].set_title( 'tSNE Projection (clusters = {0})'.format(num_clusters_smart)) ax[1].set_xticklabels([]) ax[1].set_yticklabels([]) ax[2].plot(num_clusters_list, sil_score_list_bank, 'b-', label='Silhouette', linewidth=1) ax[2].plot(num_clusters_list, cal_har_score_list_bank, 'r--', label='Calinksi-Harabasz / 5d00', linewidth=1) ax[2].plot(num_clusters_list, davies_bouldin_score_list_bank, 'g-.', label='Davies-Bouldin / 5', linewidth=1) ax[2].set(xlabel='K (# of clusters)', ylabel='Scores') ax[2].set_title('Clustering Scores') ax[2].legend() k_means = KMeans(n_clusters=num_clusters_bank, random_state=27) k_means.fit(X_train_bank) prediction_bank = k_means.predict(X_train_bank) tsne_bank = TSNEVisualizer(decompose_by=X_train_bank.shape[1] - 1, ax=ax[3], random_state=27) tsne_bank.fit(X_train_bank, ["c{}".format(c) for c in prediction_bank]) ax[3].set_title( 'tSNE Projection (clusters = {0})'.format(num_clusters_bank)) ax[3].set_xticklabels([]) ax[3].set_yticklabels([]) plt.show() # Boosting validation # Smart grid boosting_learner = AdaBoostClassifier(learning_rate=1, n_estimators=100) boost_fit_t = time() boosting_learner.fit(X_train_smart, y_train_smart) boost_fit_time = time() - boost_fit_t print('Boosting baseline fit time (smart): ' + str(boost_fit_time)) boost_pred_t = time() boost_pred = boosting_learner.predict(X_test_smart) boost_pred_time = time() - boost_pred_t print('Boosting baseline predict time (smart): ' + str(boost_pred_time)) boost_score = cross_val_score(boosting_learner, X_train_smart, y_train_smart, cv=10) print('Boosting baseline cross validation score (smart): ' + str(np.mean(boost_score))) # boost_accuracy = accuracy(boosting_learner, y_test, boost_pred) # print('Boosting baseline test set predict accuracy: ' + str(boost_accuracy)) boosting_learner = AdaBoostClassifier(learning_rate=1, n_estimators=100) boost_fit_t = time() boosting_learner.fit(X_train_smart, prediction_smart) boost_fit_time = time() - boost_fit_t print('Boosting DR + cluster fit time (smart): ' + str(boost_fit_time)) boost_pred_t = time() boost_pred = boosting_learner.predict(X_test_smart) boost_pred_time = time() - boost_pred_t print('Boosting DR + cluster predict time (smart): ' + str(boost_pred_time)) boost_score = cross_val_score(boosting_learner, X_train_smart, prediction_smart, cv=10) print('Boosting DR + cluster cross validation score (smart): ' + str(np.mean(boost_score))) # Bank loan boosting_learner = AdaBoostClassifier(learning_rate=1, n_estimators=100) boost_fit_t = time() boosting_learner.fit(X_train_bank, y_train_bank) boost_fit_time = time() - boost_fit_t print('Boosting baseline fit time (bank): ' + str(boost_fit_time)) boost_pred_t = time() boost_pred = boosting_learner.predict(X_test_bank) boost_pred_time = time() - boost_pred_t print('Boosting baseline predict time (bank): ' + str(boost_pred_time)) boost_score = cross_val_score(boosting_learner, X_train_bank, y_train_bank, cv=10) print('Boosting baseline cross validation score (bank): ' + str(np.mean(boost_score))) boosting_learner = AdaBoostClassifier(learning_rate=1, n_estimators=100) boost_fit_t = time() boosting_learner.fit(X_train_bank, prediction_bank) boost_fit_time = time() - boost_fit_t print('Boosting DR + cluster fit time (bank): ' + str(boost_fit_time)) boost_pred_t = time() boost_pred = boosting_learner.predict(X_test_bank) boost_pred_time = time() - boost_pred_t print('Boosting DR + cluster predict time (bank): ' + str(boost_pred_time)) boost_score = cross_val_score(boosting_learner, X_train_bank, prediction_bank, cv=10) print('Boosting DR + cluster cross validation score (bank): ' + str(np.mean(boost_score))) return
def load_corpus(): c = Corpus("all_posts01.txt") return c corpus = load_corpus() #tfidf = TfidfVectorizer(stop_words='english') from sklearn.cluster import KMeans vectorizer = TfidfVectorizer(max_df=0.5, max_features=10000, min_df=2, use_idf=True) #transformer = TfidfTransformer() #tfidf = make_pipeline(hasher,transformer) docs = vectorizer.fit_transform(corpus.documents) print(docs) true_k = 500 model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1) model.fit(docs) print("Top terms per cluster:") order_centroids = model.cluster_centers_.argsort()[:, ::-1] terms = vectorizer.get_feature_names() tsne = TSNEVisualizer(labels=["documents"]) tsne.fit(docs) tsne.poof()
color = "#000000" colormap.append(color) for label in labels: big_colormap.append(mycolormap[label]) t6 = time.time() tsne = TSNEVisualizer(colormap='RdYlGn') tsne.fit(tfidf_matrix, labels) tsne.poof() t7 = time.time() print("time for TSNE and vis: " + str(t7-t6)) tsne.poof()
liste_galaxies = get_list_galaxie(path) matrix = np.zeros([len(t), len(liste_galaxies)]) dirGalaxies = shelve.open(path + '/BDs/listeGalaxies') for galaxie in range(len(liste_galaxies)): for node in dirGalaxies[str(liste_galaxies[galaxie])]: matrix[index[node]][galaxie] += 1 matrix[:,galaxie] = matrix[:,galaxie] / len(dirGalaxies[str(liste_galaxies[galaxie])]) dirGalaxies.close() label = np.array([i for i in range(len(t))]) tsne = TSNEVisualizer(decompose='svd',decompose_by=15) tsne.fit(matrix, label) print(tsne.transformer_) tsne.poof() svd = TruncatedSVD(n_components=15) svd_matrix = svd.fit_transform(matrix) tsne = ts.TSNE() y = tsne.fit_transform(svd_matrix) kmeans = Kmeans(5,200,0.1) kmeans.fit(y) for i in range(kmeans.nb_cluster): print("Cluster ",i) print((np.where(kmeans.which_cluster == i))[0]) print() plt.scatter(y[:, 0], y[:, 1], c=kmeans.which_cluster.reshape(-1,1), s=50, cmap='viridis')
random_state=random_state, verbose=1, n_jobs=num_threads) kmeans = kmeans.fit(X) pickle.dump(kmeans, open(f"cache/{thread_id}/{thread_id}_fitted_kmeans.pk", 'wb')) y = kmeans.labels_ if os.path.exists(f"cache/{thread_id}/{thread_id}_tsne_fitted.pk"): print(f"Loading pre-trained tsne comments.") tsne = pickle.load( open(f"cache/{thread_id}/{thread_id}_tsne_fitted.pk", 'rb')) else: tsne = TSNEVisualizer(decompose_by=tsne_svd, n_iter=tsne_iterations, verbose=2) tsne = tsne.fit(X, y) pickle.dump(tsne, open(f"cache/{thread_id}/{thread_id}_tsne_fitted.pk", 'wb')) tsne.colors = """#33000e, #660029, #bf0080, #660080, #0088ff, #00708c, #008066, #4cbf00, #735c00, #ff8800, #995200, #402200, #ff4400, #590000, #ff4073, #ff40f2, #7736d9, #101040, #233f8c, #36ced9, #36d98d, #538020, #b6bf30, #b22d2d, #733960, #8959b3, #1a2033, #46628c, #73bfe6, #1a2e33, #204020, #ffd580, #f29979, #8c5946, #cc99c2, #bfbfff, #698c8a, #eaffbf, #8c8569, #4d4439, #bfa38f, #e6acac""".split( ', ') tsne.draw(tsne.vecs, y, point_annotations=[ f"{document}\n{author}" for document, author in zip(documents, authors) ]) with open(f'cache/{thread_id}/{thread_id}_clustered.tsv', 'w') as file:
from src.utils.input_reader import * # LOAD DATA path_prefix = ".." + os.sep + ".." + os.sep + "input" + os.sep train_df, test_df, sample_df = load_data_sets(path_prefix + "train.csv", path_prefix + "test.csv", None) xtrain, xtest, ytrain, ytest = train_vali_split(train_df) print('average sentence length: %s' % train_df['text'].map(lambda x: len(x.split())).mean()) print('min sentence length: %s' % train_df['text'].map(lambda x: len(x.split())).min()) # Load the data and create document vectors tfv = TfidfVectorizer(min_df=3, max_features=None, strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}', ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1) docs = tfv.fit_transform(list(xtrain) + list(xtest)) labels = list(ytrain) + list(ytest) # Create the visualizer and draw the vectors tsne = TSNEVisualizer(labels=['Author A', 'Author B', 'Author C']) tsne.fit(docs, labels) tsne.poof()
def TSNE_graph(X_train, y_train): tsne = TSNEVisualizer() tsne.fit(X_train, y_train) tsne.poof()
from sklearn.feature_extraction.text import CountVectorizer vect = CountVectorizer(tokenizer=lambda x: [i.strip() for i in x.split(',')], lowercase=False) dummies = vect.fit_transform(df['ingredients'].apply(','.join)) df = pd.DataFrame(dummies.todense(),columns=vect.get_feature_names()) print("Vocab Length: ", len(vect.get_feature_names())) print("All Data Shape: ", df.shape) df.index= df_index print("Number of Predictors: ", df.shape[0]) df.head() # Create the visualizer and draw the vectors plt.figure(figsize = [15,9]) tsne = TSNEVisualizer() tsne.fit(df.loc[traindex,:][:7000], y[:7000]) tsne.poof() X = df.loc[traindex,:] print("Number of Cuisine Types: ", y.nunique()) print("X Shape: ", X.shape) test_df = df.loc[testdex,:] print("Test DF Shape: ", test_df.shape) del df; gc.collect(); LogisticRegression().get_params().keys() model = LogisticRegression(multi_class= 'ovr') score = cross_validate(model, X, y, return_train_score=False) score["test_score"].mean()
def initialization_layers_train(self, train_path, test_path, dataset, no_of_layers, filter_sizes): global n1, n2 # y_pred_cl=tf.get_variable("y_pred_cl",0) x, x_image, y_true, y_true_cls = self.load_main(train_path, dataset, 0) self.load_main(test_path, dataset, 1) # print(len(x_image)) # y_true_cls=tf.Variable(y_true_cls) if dataset == "Fashion-MNIST": layer_conv1, weights_conv1 = self.new_conv_layer( input=x_image, num_input_channels=1, filter_size=filter_sizes[0], num_filters=64, name="conv1") elif dataset == "CIFAR-10": layer_conv1, weights_conv1 = self.new_conv_layer( input=x_image, num_input_channels=3, filter_size=filter_sizes[0], num_filters=64, name="conv1") layer_pool1 = self.new_pool_layer(layer_conv1, name="pool1") layer_pool1 = tf.nn.local_response_normalization(layer_pool1) layer_relu1 = self.new_relu_layer(layer_pool1, name="relu1") layer_pools = [] layer_relus = [] layer_convs = [] weight_convs = [] layer_convs.append(layer_conv1) layer_pools.append(layer_pool1) layer_relus.append(layer_relu1) weight_convs.append(weights_conv1) n2 = 5 for k1 in range(1, no_of_layers): namee = "conv" + str(k1 + 1) layer_conv1, weights_conv1 = self.new_conv_layer( input=layer_relus[k1 - 1], num_input_channels=64, filter_size=filter_sizes[k1], num_filters=64, name=namee) name2 = "pool" + str(k1 + 1) name1 = "relu" + str(k1 + 1) layer_pool1 = self.new_pool_layer(layer_conv1, name=name2) layer_pool1 = tf.nn.local_response_normalization(layer_pool1) layer_relu1 = self.new_relu_layer(layer_pool1, name=name1) layer_convs.append(layer_conv1) layer_pools.append(layer_pool1) layer_relus.append(layer_relu1) weight_convs.append(weights_conv1) n2 = 10 num_features = layer_relu1.get_shape()[1:4].num_elements() layer_flat = tf.reshape(layer_relu1, [-1, num_features]) layer_fc1 = self.new_fc_layer(layer_flat, num_inputs=num_features, num_outputs=512, name="fc1") layer_relu4 = self.new_relu_layer(layer_fc1, name="relu" + str(no_of_layers + 1)) layer_fc3 = self.new_fc_layer(layer_relu4, num_inputs=512, num_outputs=192, name="fc3") layer_relu3 = self.new_relu_layer(layer_fc3, name="relu" + str(no_of_layers + 2)) layer_fc2 = self.new_fc_layer(input=layer_relu3, num_inputs=192, num_outputs=10, name="fc2") with tf.variable_scope("Softmax"): y_pred = (tf.nn.softmax(layer_fc2)) y_pred_cls = tf.argmax(y_pred, dimension=1) # y_pred_cl=y_pred_cls with tf.name_scope("cross_ent"): cross_entropy = tf.nn.softmax_cross_entropy_with_logits( logits=layer_fc2, labels=y_true) cost = tf.reduce_mean(cross_entropy) with tf.name_scope("optimizer"): optimizer = tf.train.AdamOptimizer( learning_rate=1e-3).minimize(cost) with tf.variable_scope("accuracy"): # print("-----------") # print(y_pred_cls) correct_prediction = tf.equal(y_pred_cls, y_true_cls) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) num_epochs = 20 batch_size = 100 trainset = self.trainset testset = self.testset trainlabel = self.trainlabel testlabel = self.testlabel f = open("CF_allembeddings.txt", "w") tsne = TSNEVisualizer() with tf.Session() as sess: global train_acc global f1_mi global f1_ma train_acc = [] f1_mi = [] f1_ma = [] sess.run(tf.global_variables_initializer()) trainset1 = self.trainset trainlabels11 = self.train_labels # print(len(trainset1)) trainlabel1 = self.trainlabel for i in range(0, 4): f.write("\n\n Percentage of traindata ::: ") f.write(str((1 + i) * 10)) f.write("\n") # indx = np.arange(trainlabel1.shape[0]) # np.random.shuffle(indx) # trainset,trainlabel = trainset1[indx], trainlabel1[indx] # trainset1, trainlabel1=trainset1[indx], trainlabel1[indx] # trainlabels1=trainlabels11[indx] n = len(trainset1) n = int(n * (i + 1) / 10) # print(n) trainset, trainlabel = trainset1, trainlabel1 trainlabels1 = trainlabels11 testset, testlabel = trainset[n:, :], trainlabel[n:, :] testlabels1 = trainlabels11[n:, :] # print(len(testlabel)) trainset, trainlabel = trainset[0:n, :], trainlabel[0:n, :] trainlabels1 = trainlabels1[0:n, :] print("----------------------------------------------") # print(len(trainlabel1),len(trainset),len(testset)) # testset,testlabel =trainset1[n:,:],trainlabel1[n:,:] # print(trainlabel) # writer.add_graph(sess.graph) # print("testlabel:",str(testlabels1[4])) # print("testsset",str(testset[4])) # print("testlabel",str(testlabel[4])) # print("set",str(trainset1[n+4])) # print("lab",str(trainlabel1[n+4])) # break for epoch in range(num_epochs): start_time = time.time() train_accuracy = 0 k = 0 f1_macro = 0 f1_micro = 0 a1 = 0 # batch_size=int(len(trainlabel)) for batch in range(0, int(len(trainlabel) / batch_size)): # print("--") x_batch = trainset[k:k + batch_size] y_true_batch = trainlabel[k:k + batch_size] k = k + batch_size feed_dict_train = {x: x_batch, y_true: y_true_batch} sess.run(optimizer, feed_dict=feed_dict_train) a, y_p, y, t = sess.run( [accuracy, y_pred_cls, y_true_cls, layer_relu3], feed_dict=feed_dict_train) a1 += a # a1+=a print("acc :", str(epoch), "---", str(a1 / int(len(trainlabel) / batch_size))) vali_accuracy, l = sess.run([accuracy, layer_relu3], feed_dict={ x: testset, y_true: testlabel }) # print(np.array(l)) kmeans = KMeans(n_clusters=10, random_state=0).fit(np.array(l)) # yy=kmeans.predict(np.array(l)) # plt.scatter() labels = kmeans.labels_ # print(labels) f.write("\n\nEMBEDDINGS ARE : \n\n") f.write(str(np.array(l))) f.write("\n\n") mydict = { i: np.where(kmeans.labels_ == i)[0] for i in range(kmeans.n_clusters) } # print(mydict) # print(len(mydict[9])) # print(len(mydict[8])) # print(len(mydict[7])) # print(len(mydict[6])) dictlist = [] #print(mydict) mydict1 = {} for key in mydict: mydict1[key] = list(mydict[key]) # print(len(mydict[0])) cluster_label = {} acc = 0.0 # print(testlabels1[0][0]) lab = [i for i in range(10)] while (len(lab) != 0): #iidx=max(mydict.items(), key=operator.itemgetter(1))[0] #idx is the cluster no. iidx = max(mydict1, key=mydict1.get) cluster_label[iidx] = 0 keys1 = mydict1[iidx] # print(keys1) dict1 = {} print(keys1) for i in (keys1): if testlabels1[i][0] not in dict1: dict1[testlabels1[i][0]] = 0 dict1[testlabels1[i][0]] += 1 idx1 = max( dict1.items(), key=operator.itemgetter(1) )[0] #idx1 is the label of maximum occuring samples in idx cluste while (idx1 not in lab): del dict1[idx1] idx1 = max(dict1.items(), key=operator.itemgetter(1))[0] # print(dict1[idx1]) # print(dict1) #print(cluster_label) acc += (dict1[idx1]) cluster_label[iidx] = idx1 # print(idx1) lab.remove(idx1) del mydict1[iidx] print("Acc: ") print(acc) print(acc / len(testset)) f.write("\nACCURACY AFTER CLUSTERING IS: ") f.write(str(acc)) f.write("\n\n")