def q6(): data_table = viz.load_data_table(viz.DATA_111_URL) singleton_list=[] for line in data_table: singleton_list.append(alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) alg_clusters_matplotlib.plot_clusters(data_table, cluster_list, True)
def run_example(): """ Load a data table, compute a list of clusters and plot a list of clusters Set DESKTOP = True/False to use either matplotlib or simplegui """ #data_table = load_data_table(DATA_3108_URL) #data_table = load_data_from_file("unifiedCancerData_290.csv") #data_table = load_data_from_file("unifiedCancerData_896.csv") #data_table = load_data_from_file("unifiedCancerData_3108.csv") data_table = load_data_from_file("unifiedCancerData_111.csv") singleton_list = gen_singleton_list(data_table) #print alg_project3.fast_closest_pair(singleton_list) #print alg_project3.slow_closest_pairs(singleton_list) #cluster_list = sequential_clustering(singleton_list, 15) #print "Displaying", len(cluster_list), "sequential clusters" cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) print "Displaying", len(cluster_list), "k-means clusters" print "compute_distortion : ", alg_project3.compute_distortion( cluster_list, data_table) cluster_list = alg_project3.hierarchical_clustering(singleton_list, 9) print "Displaying", len(cluster_list), "hierarchical clusters" print "compute_distortion : ", alg_project3.compute_distortion( cluster_list, data_table)
def run_example(): """ Load a data table, compute a list of clusters and plot a list of clusters Set DESKTOP = True/False to use either matplotlib or simplegui """ #data_table = load_data_table(DATA_3108_URL) #data_table = load_data_from_file("unifiedCancerData_290.csv") #data_table = load_data_from_file("unifiedCancerData_896.csv") #data_table = load_data_from_file("unifiedCancerData_3108.csv") data_table = load_data_from_file("unifiedCancerData_111.csv") singleton_list = gen_singleton_list(data_table) #print alg_project3.fast_closest_pair(singleton_list) #print alg_project3.slow_closest_pairs(singleton_list) #cluster_list = sequential_clustering(singleton_list, 15) #print "Displaying", len(cluster_list), "sequential clusters" cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) print "Displaying", len(cluster_list), "k-means clusters" print "compute_distortion : ", alg_project3.compute_distortion(cluster_list, data_table) cluster_list = alg_project3.hierarchical_clustering(singleton_list, 9) print "Displaying", len(cluster_list), "hierarchical clusters" print "compute_distortion : ", alg_project3.compute_distortion(cluster_list, data_table)
def q6(): data_table = viz.load_data_table(viz.DATA_111_URL) singleton_list = [] for line in data_table: singleton_list.append( alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) alg_clusters_matplotlib.plot_clusters(data_table, cluster_list, True)
def q10(): nodes_list = {viz.DATA_111_URL:111, viz.DATA_290_URL:290, viz.DATA_896_URL:896} url_list = [viz.DATA_111_URL, viz.DATA_290_URL, viz.DATA_896_URL] kmeans_dict = dict() hierarchical_dict = dict() for url in url_list: data_table = viz.load_data_table(url) singleton_list = [] for line in data_table: singleton_list.append(alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) kmeans_dict[url] = list() hierarchical_dict[url] = list() cluster_range = range(6, 20 + 1) for cluster_count in cluster_range: #kmeans cluster_list = alg_project3.kmeans_clustering(singleton_list, cluster_count, 5) kmeans_error = compute_distortion(cluster_list, data_table) kmeans_dict[url].append(kmeans_error) #hierarchical count = 20 while count >= 6: alg_project3.hierarchical_clustering(singleton_list, count) hierarchical_error = compute_distortion(singleton_list, data_table) hierarchical_dict[url].insert(0, hierarchical_error) count -= 1 for url in url_list: plt.title('Distortion for hierarchical and k-means clustering for '+str(nodes_list[url])+' points') plt.xlabel('Number of clusters') plt.ylabel('Distortion') line1, = plt.plot(cluster_range, kmeans_dict[url],'g') line2, = plt.plot(cluster_range, hierarchical_dict[url],'b') plt.legend((line1, line2), ('kmeans clustering', 'hierarchical clustering')) plt.show() #q2() #q2() #q3() #q5() #q6() #q7() #q10()
def q7(): data_table = viz.load_data_table(viz.DATA_111_URL) singleton_list = [] for line in data_table: singleton_list.append(alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) error2 = compute_distortion(cluster_list, data_table) cluster_list = alg_project3.hierarchical_clustering(singleton_list, 9) error1 = compute_distortion(cluster_list, data_table) print 'hierarchical clustering',error1 print 'kmeans clustering', error2
def q10(): nodes_list = { viz.DATA_111_URL: 111, viz.DATA_290_URL: 290, viz.DATA_896_URL: 896 } url_list = [viz.DATA_111_URL, viz.DATA_290_URL, viz.DATA_896_URL] kmeans_dict = dict() hierarchical_dict = dict() for url in url_list: data_table = viz.load_data_table(url) singleton_list = [] for line in data_table: singleton_list.append( alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) kmeans_dict[url] = list() hierarchical_dict[url] = list() cluster_range = range(6, 20 + 1) for cluster_count in cluster_range: #kmeans cluster_list = alg_project3.kmeans_clustering( singleton_list, cluster_count, 5) kmeans_error = compute_distortion(cluster_list, data_table) kmeans_dict[url].append(kmeans_error) #hierarchical count = 20 while count >= 6: alg_project3.hierarchical_clustering(singleton_list, count) hierarchical_error = compute_distortion(singleton_list, data_table) hierarchical_dict[url].insert(0, hierarchical_error) count -= 1 for url in url_list: plt.title('Distortion for hierarchical and k-means clustering for ' + str(nodes_list[url]) + ' points') plt.xlabel('Number of clusters') plt.ylabel('Distortion') line1, = plt.plot(cluster_range, kmeans_dict[url], 'g') line2, = plt.plot(cluster_range, hierarchical_dict[url], 'b') plt.legend((line1, line2), ('kmeans clustering', 'hierarchical clustering')) plt.show()
def q7(): data_table = viz.load_data_table(viz.DATA_111_URL) singleton_list = [] for line in data_table: singleton_list.append( alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) cluster_list = alg_project3.kmeans_clustering(singleton_list, 9, 5) error2 = compute_distortion(cluster_list, data_table) cluster_list = alg_project3.hierarchical_clustering(singleton_list, 9) error1 = compute_distortion(cluster_list, data_table) print('hierarchical clustering', error1) print('kmeans clustering', error2)
def test_kmeans(): """ Test for k-means clustering kmeans_clustering should not mutate cluster_list, but make a new copy of each test anyways """ # load small data table print print "Testing kmeans_clustering on 24 county set" data_24_table = load_data_table(DATA_24_URL) kmeansdata_24 = [[15, 1, set([('34017', '36061'), ('06037',), ('06059',), ('36047',), ('36081',), ('06071', '08031'), ('36059',), ('36005',), ('55079',), ('34013', '34039'), ('06075',), ('01073',), ('06029',), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')])], [15, 3, set([('34017', '36061'), ('06037', '06059'), ('06071',), ('36047',), ('36081',), ('08031',), ('36059',), ('36005',), ('55079',), ('34013', '34039'), ('06075',), ('01073',), ('06029',), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')])], [15, 5, set([('34017', '36061'), ('06037', '06059'), ('06071',), ('36047',), ('36081',), ('08031',), ('36059',), ('36005',), ('55079',), ('34013', '34039'), ('06075',), ('01073',), ('06029',), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')])], [10, 1, set([('34017', '36061'), ('06029', '06037', '06075'), ('11001', '24510', '34013', '34039', '51013', '51760', '51840', '54009'), ('06059',), ('36047',), ('36081',), ('06071', '08031', '41051', '41067'), ('36059',), ('36005',), ('01073', '55079')])], [10, 3, set([('34013', '34017', '36061'), ('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('34039', '36047'), ('36081',), ('36059',), ('36005',), ('01073', '55079'), ('11001', '24510', '51013', '51760', '51840', '54009')])], [10, 5, set([('34013', '34017', '36061'), ('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('34039', '36047'), ('36081',), ('36059',), ('36005',), ('01073', '55079'), ('11001', '24510', '51013', '51760', '51840', '54009')])], [5, 1, set([('06029', '06037', '06075'), ('01073', '11001', '24510', '34013', '34017', '34039', '36047', '51013', '51760', '51840', '54009', '55079'), ('06059',), ('36005', '36059', '36061', '36081'), ('06071', '08031', '41051', '41067')])], [5, 3, set([('06029', '06037', '06075'), ('11001', '24510', '34013', '34017', '34039', '36005', '36047', '36059', '36061', '36081', '51013'), ('08031', '41051', '41067'), ('06059', '06071'), ('01073', '51760', '51840', '54009', '55079')])], [5, 5, set([('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('01073', '55079'), ('11001', '24510', '34013', '34017', '34039', '36005', '36047', '36059', '36061', '36081', '51013', '51760', '51840', '54009')])]] suite = poc_simpletest.TestSuite() for num_clusters, num_iterations, expected_county_tuple in kmeansdata_24: # build initial list of clusters for each test since mutation is allowed cluster_list = [] for idx in range(len(data_24_table)): line = data_24_table[idx] cluster_list.append(alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) # compute student answer student_clustering = student.kmeans_clustering(cluster_list, num_clusters, num_iterations) student_county_tuple = set_of_county_tuples(student_clustering) # Prepare test error_message = "Testing kmeans_custering on 24 county table, num_clusters = " + str(num_clusters) error_message += " num_iterations = " + str(num_iterations) error_message += "\nStudent county tuples: " + str(student_county_tuple) error_message += "\nExpected county tuples: " + str(expected_county_tuple) suite.run_test(student_county_tuple == expected_county_tuple, True, error_message) suite.report_results()
def run_example(): """ Load a data table, compute a list of clusters and plot a list of clusters Set DESKTOP = True/False to use either matplotlib or simplegui """ data_table = load_data_table(DATA_896_URL) singleton_list = [] for line in data_table: singleton_list.append(alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) #cluster_list = sequential_clustering(singleton_list, 15) #print "Displaying", len(cluster_list), "sequential clusters" k_dis = [] h_dis = [] for cluster_num in xrange(6, 21): cluster_list = alg_project3.hierarchical_clustering([c.copy() for c in singleton_list], cluster_num) print "Displaying", len(cluster_list), "hierarchical clusters" h_dis.append(alg_app3.compute_distortion(cluster_list, data_table)) cluster_list = alg_project3.kmeans_clustering([c.copy() for c in singleton_list], cluster_num, 5) print "Displaying", len(cluster_list), "k-means clusters" k_dis.append(alg_app3.compute_distortion(cluster_list, data_table)) xvals = range(6, 21) plt.plot(xvals, k_dis, '-b', label='kmeans_clustering (5 iteration)') plt.plot(xvals, h_dis, '-r', label='hierarchical_clustering') plt.legend(loc='upper right') plt.xlabel("number of clusters") plt.ylabel("distortion") plt.title("Distortion of 2 clustering methods: 896 counties") plt.show() # draw the clusters using matplotlib or simplegui """
def test_kmeans(): """ Test for k-means clustering kmeans_clustering should not mutate cluster_list, but make a new copy of each test anyways """ # load small data table print print "Testing kmeans_clustering on 24 county set" data_24_table = load_data_table(DATA_24_URL) kmeansdata_24 = [[ 15, 1, set([('34017', '36061'), ('06037', ), ('06059', ), ('36047', ), ('36081', ), ('06071', '08031'), ('36059', ), ('36005', ), ('55079', ), ('34013', '34039'), ('06075', ), ('01073', ), ('06029', ), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')]) ], [ 15, 3, set([('34017', '36061'), ('06037', '06059'), ('06071', ), ('36047', ), ('36081', ), ('08031', ), ('36059', ), ('36005', ), ('55079', ), ('34013', '34039'), ('06075', ), ('01073', ), ('06029', ), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')]) ], [ 15, 5, set([('34017', '36061'), ('06037', '06059'), ('06071', ), ('36047', ), ('36081', ), ('08031', ), ('36059', ), ('36005', ), ('55079', ), ('34013', '34039'), ('06075', ), ('01073', ), ('06029', ), ('41051', '41067'), ('11001', '24510', '51013', '51760', '51840', '54009')]) ], [ 10, 1, set([('34017', '36061'), ('06029', '06037', '06075'), ('11001', '24510', '34013', '34039', '51013', '51760', '51840', '54009'), ('06059', ), ('36047', ), ('36081', ), ('06071', '08031', '41051', '41067'), ('36059', ), ('36005', ), ('01073', '55079')]) ], [ 10, 3, set([('34013', '34017', '36061'), ('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('34039', '36047'), ('36081', ), ('36059', ), ('36005', ), ('01073', '55079'), ('11001', '24510', '51013', '51760', '51840', '54009')]) ], [ 10, 5, set([('34013', '34017', '36061'), ('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('34039', '36047'), ('36081', ), ('36059', ), ('36005', ), ('01073', '55079'), ('11001', '24510', '51013', '51760', '51840', '54009')]) ], [ 5, 1, set([('06029', '06037', '06075'), ('01073', '11001', '24510', '34013', '34017', '34039', '36047', '51013', '51760', '51840', '54009', '55079'), ('06059', ), ('36005', '36059', '36061', '36081'), ('06071', '08031', '41051', '41067')]) ], [ 5, 3, set([('06029', '06037', '06075'), ('11001', '24510', '34013', '34017', '34039', '36005', '36047', '36059', '36061', '36081', '51013'), ('08031', '41051', '41067'), ('06059', '06071'), ('01073', '51760', '51840', '54009', '55079')]) ], [ 5, 5, set([('06029', '06037', '06075'), ('08031', '41051', '41067'), ('06059', '06071'), ('01073', '55079'), ('11001', '24510', '34013', '34017', '34039', '36005', '36047', '36059', '36061', '36081', '51013', '51760', '51840', '54009')]) ]] suite = poc_simpletest.TestSuite() for num_clusters, num_iterations, expected_county_tuple in kmeansdata_24: # build initial list of clusters for each test since mutation is allowed cluster_list = [] for idx in range(len(data_24_table)): line = data_24_table[idx] cluster_list.append( alg_cluster.Cluster(set([line[0]]), line[1], line[2], line[3], line[4])) # compute student answer student_clustering = student.kmeans_clustering(cluster_list, num_clusters, num_iterations) student_county_tuple = set_of_county_tuples(student_clustering) # Prepare test error_message = "Testing kmeans_custering on 24 county table, num_clusters = " + str( num_clusters) error_message += " num_iterations = " + str(num_iterations) error_message += "\nStudent county tuples: " + str( student_county_tuple) error_message += "\nExpected county tuples: " + str( expected_county_tuple) suite.run_test(student_county_tuple == expected_county_tuple, True, error_message) suite.report_results()