def run_simulated_manually(filepath, lines_to_read=sys.maxint, cluster_args=None): with open(get_resource_name(filepath), 'r') as f: data = [line.rstrip().split(' ') for line in islice(f, lines_to_read)] if cluster_args is None: cluster_args = dict() cluster = Cluster(**cluster_args) shingler = Shingler(span=3) s = FeatureClusterSummarizer() content_dict = dict() for pair in data: if len(pair) > 1: label, text = pair else: label, text = pair[0], '' content_dict[label] = text shingles = shingler.get_shingles(text) s.add_features(label, shingles) cluster.add_set(shingles, label) clusters = cluster.get_clusters() is_label_positive = lambda lbl: ':' in lbl return dict(stats=get_stats(clusters, is_label_positive), uindex=s.summarize_clusters(clusters))
def test_cluster_threshold(self): """Expected error for threshold to similarity should be reasonable""" n_tests = 50 dim = 15 expected_error = 0.20 tot_err = 0 for test in range(n_tests): # Get some sets and their similarities sets = (randset(), randset()) jsim = jaccard_sim(*sets) # Find the threshold at which they cluster together for threshold in range(1, 100, 5): threshold = float(threshold) / 100 bandwidth = get_bandwidth(dim, threshold) num_bands = int(dim / bandwidth) cluster = Cluster(width=num_bands * bandwidth, bandwidth=bandwidth) cluster.add_set(sets[0]) cluster.add_set(sets[1]) if len(cluster.get_clusters()) == 2: tot_err += abs(jsim - threshold) break avg_err = float(tot_err) / n_tests self.assertLessEqual(avg_err, expected_error, "Error too large")
def test_same_set(self): """A set should be clustered with itself""" s = randset() cluster = Cluster(width=10, bandwidth=2) cluster.add_set(s) cluster.add_set(s) self.assertEqual(len(cluster.get_clusters()), 1)
def test_dissimilar_sets(self): """Two non-similar sets should not be clustered""" cluster = Cluster(width=10, bandwidth=2) cluster.add_set("12yu5abcdef") cluster.add_set("1234567890z") print cluster.get_clusters() num_clusters = len(cluster.get_clusters()) self.assertEqual(num_clusters, 2, "Expected 2 clusters, got {}".format(num_clusters))
def test_names(self): """Should return 281 clusters of names. """ with open(get_resource_name('data/perrys.csv'), 'r') as f: data = set(line.rstrip() for line in f) cluster = Cluster(width=20, bandwidth=5) shingler = Shingler(3) s = FeatureClusterSummarizer() for name in data: shingles = shingler.get_shingles(name) s.add_features(name, shingles) cluster.add_set(shingles, name) clusters = cluster.get_clusters() ti = s.summarize_clusters(clusters) self.assertEqual(len(clusters), 281) self.assertAlmostEqual(ti, 0.9780512134223747)
def test_bills(self): """Should return 97 clusters of bills. """ with open(get_resource_name('data/bills100.txt'), 'r') as f: data = [line.rstrip().split('|') for line in f] cluster = Cluster(width=20, bandwidth=5) shingler = Shingler(span=3, tokenizer=RegexTokenizer()) s = FeatureClusterSummarizer() for label, text in data: shingles = shingler.get_shingles(text) s.add_features(label, shingles) cluster.add_set(shingles, label) clusters = cluster.get_clusters() ti = s.summarize_clusters(clusters) self.assertEqual(len(clusters), 97) self.assertAlmostEqual(ti, 1.0)
def test_names_kmin(self): """Should return 252 clusters of names. """ with open(get_resource_name('data/perrys.csv'), 'r') as f: data = set(line.rstrip() for line in f) cluster = Cluster(width=20, bandwidth=5, kmin=2) shingler = Shingler(3) s = FeatureClusterSummarizer() for name in data: shingles = shingler.get_shingles(name) s.add_features(name, shingles) cluster.add_set(shingles, name) clusters = cluster.get_clusters() # for cluster in clusters: # print cluster ti = s.summarize_clusters(clusters) self.assertEqual(len(clusters), 252) self.assertAlmostEqual(ti, 0.9732840816954408)
def test_empty(self): """Should place the two empty sets into a separate cluster""" cluster = Cluster(width=10, bandwidth=2) cluster.add_set("abcdefg") cluster.add_set("abcdefghi") cluster.add_set("") cluster.add_set("") self.assertEqual(len(cluster.get_clusters()), 2)
def test_simulated(opts, data): cluster = Cluster(width=opts.width, bandwidth=opts.bandwidth, lsh_scheme=opts.lsh_scheme) shingler = Shingler(span=opts.shingle_span) s = FeatureClusterSummarizer() content_dict = dict() for label, text in data: content_dict[label] = text shingles = shingler.get_shingles(text) s.add_features(label, shingles) cluster.add_set(shingles, label) clusters = cluster.get_clusters() c = get_stats(clusters, lambda x: len(x.split(':')) > 1) ti = s.summarize_clusters(clusters) print json.dumps(dict( stats=c.dict(), ratios=dict( precision=c.get_precision(), recall=c.get_recall() ), ti=ti ))
def test_similar_sets(self): """Two similar sets should be clustered""" cluster = Cluster(width=10, bandwidth=2) cluster.add_set("abcdefg") cluster.add_set("abcdefghi") self.assertEqual(len(cluster.get_clusters()), 1)