def concept_clustering(data, skill, cluster_number=3, plot=True): pk, level = data.get_skill_id(skill) items = data.get_items_df() items = items[items["skill_lvl_" + str(level)] == pk] skills = data.get_skills_df() skill_ids = items[~items["skill_lvl_3"].isnull()]["skill_lvl_3"].unique() corr = compute_corr(data, merge_skills=True) corr = pd.DataFrame(corr, index=skill_ids, columns=skill_ids) print("Corr ({}) contain total {} values and from that {} nans".format(corr.shape, corr.size, corr.isnull().sum().sum())) corr[corr.isnull()] = 0 try: sc = SpectralClusterer(corr, kcut=corr.shape[0] * 0.5, mutual=True) labels = sc.run(cluster_number=cluster_number, KMiter=50, sc_type=2) except np.linalg.linalg.LinAlgError: sc = SpectralClusterer(corr, kcut=corr.shape[0] * 0.5, mutual=False) labels = sc.run(cluster_number=cluster_number, KMiter=50, sc_type=2) if plot: colors = "rgbyk" for i, p in enumerate(corr.columns): skill = skills.loc[int(p)] plt.plot(sc.eig_vect[i, 1], sc.eig_vect[i, 2], "o", color=colors[labels[i]]) plt.text(sc.eig_vect[i, 1], sc.eig_vect[i, 2], skill["name"]) plt.title(data) return labels
def hierarchical_clustering(data, skill, method='single', metric='euclidean', dendrogram=True, concepts=False, cluster_number=3, corr_as_vectors=False): pk, level = data.get_skill_id(skill) items = data.get_items_df() skills = data.get_skills_df() corr = compute_corr(data, merge_skills=concepts) print("Corr ({}) contain total {} values and from that {} nans".format(corr.shape, corr.size, corr.isnull().sum().sum())) corr[corr.isnull()] = 0 if concepts: items = items[items["skill_lvl_" + str(level)] == pk] skill_ids = items[~items["skill_lvl_3"].isnull()]["skill_lvl_3"].unique() corr = pd.DataFrame(corr, index=skill_ids, columns=skill_ids) labels = list(skills.loc[corr.index]["name"]) else: items = items[items["skill_lvl_" + str(level)] == pk] items = items[items["visualization"] != "pairing"] corr = pd.DataFrame(corr, index=items.index, columns=items.index) labels = ["{1} - {0}".format(item["name"], item["visualization"][0]) for id, item in list(items.iterrows())] if corr_as_vectors: Z = hr.linkage(corr, method=method, metric=metric) else: Z = hr.linkage(dst.squareform(1 - corr), method=method) Z[Z < 0] = 0 if dendrogram: plt.title('{}: method: {}, metric: {}, as vectors: {}'.format(skill, method, metric, corr_as_vectors)) plt.xlabel('items' if not concepts else "concepts") plt.ylabel('distance') hr.dendrogram(Z, leaf_rotation=90., leaf_font_size=10., labels=labels) return hr.fcluster(Z, cluster_number, "maxclust")
def item_clustering(data, skill, cluster_number=3, plot=True): pk, level = data.get_skill_id(skill) items = data.get_items_df() items = items[items["skill_lvl_" + str(level)] == pk] items = items[items["visualization"] != "pairing"] corr = compute_corr(data) corr = pd.DataFrame(corr, index=items.index, columns=items.index) print("Corr ({}) contain total {} values and from that {} nans".format(corr.shape, corr.size, corr.isnull().sum().sum())) corr[corr.isnull()] = 0 sc = SpectralClusterer(corr, kcut=corr.shape[0] / 2, mutual=True) # sc = SpectralClusterer(corr, kcut=30, mutual=True) labels = sc.run(cluster_number=cluster_number, KMiter=50, sc_type=2) if plot: colors = "rgbyk" visualizations = list(items["visualization"].unique()) for i, p in enumerate(corr.columns): item = items.loc[p] plt.plot(sc.eig_vect[i,1], sc.eig_vect[i,2], "o", color=colors[visualizations.index(item["visualization"])]) # plt.plot(sc.eig_vect[i, 1], sc.eig_vect[i, 2], "o", color=colors[labels[i]]) plt.text(sc.eig_vect[i, 1], sc.eig_vect[i, 2], item["name"]) for i, vis in enumerate(visualizations): plt.plot(0, 0, "o", color=colors[i], label=vis) plt.title(data) plt.legend(loc=3) return labels