def _init_colors(self, args, headers, data): """ Colors are numeric, and use whatever gradient is built into R. """ self.color_header = args.color if self.color_header not in headers: raise ValueError('bad color column header: ' + self.color_header) index = self.h_to_i[self.color_header] try: self.color_list = Carbone.get_numeric_column(data, index) except Carbone.NumericError: raise ValueError( 'expected the color column %s ' 'to be numeric' % self.color_header)
def _init_axes(self, args, headers, data): # read the axes self.axis_headers = args.axes # verify the number of axis headers if len(self.axis_headers) != 3: raise ValueError('expected three axis column headers') # verify the axis header contents bad_axis_headers = set(self.axis_headers) - set(headers) if bad_axis_headers: raise ValueError('bad axis column headers: ' + ', '.join(bad_axis_headers)) self.axis_lists = [] for h in self.axis_headers: index = self.h_to_i[h] try: axis_list = Carbone.get_numeric_column(data, index) except Carbone.NumericError: raise ValueError('expected the axis column %s ' 'to be numeric' % h) self.axis_lists.append(axis_list)
def _init_axes(self, args, headers, data): # read the axes self.axis_headers = args.axes # verify the number of axis headers if len(self.axis_headers) != 2: raise ValueError('expected two axis column headers') # verify the axis header contents bad_axis_headers = set(self.axis_headers) - set(headers) if bad_axis_headers: raise ValueError( 'bad axis column headers: ' + ', '.join(bad_axis_headers)) self.axis_lists = [] for h in self.axis_headers: index = self.h_to_i[h] try: axis_list = Carbone.get_numeric_column(data, index) except Carbone.NumericError: raise ValueError( 'expected the axis column %s ' 'to be numeric' % h) self.axis_lists.append(axis_list)
def _init_axes(self, args, headers, data): # read the axes self.axis_headers = args.axes # verify the number of axis headers if len(self.axis_headers) != 3: raise ValueError("expected three axis column headers") # verify the axis header contents bad_axis_headers = set(self.axis_headers) - set(headers) if bad_axis_headers: msg_a = "bad axis column headers: " msg_b = ", ".join(bad_axis_headers) raise ValueError(msg_a + msg_b) self.axis_lists = [] for h in self.axis_headers: index = self.h_to_i[h] try: axis_list = Carbone.get_numeric_column(data, index) except Carbone.NumericError: msg_a = "expected the axis column %s " % h msg_b = "to be numeric" raise ValueError(msg_a + msg_b) self.axis_lists.append(axis_list)
def get_rtable_info(rtable, cluster_header, axis_headers): """ @param rtable: a RUtil.RTable object @param cluster_header: header of the new column to add @param axis_headers: a tuple of column headers @return: points as rows in a numpy array """ header_row = rtable.headers data_rows = rtable.data # do header validation Carbone.validate_headers(header_row) if not Carbone.is_valid_header(cluster_header): raise ValueError('invalid column header: %s' % cluster_header) if cluster_header in header_row: raise ValueError( 'the column header %s ' 'is already in the table' % cluster_header) # get the numpy array of conformant points h_to_i = dict((h, i+1) for i, h in enumerate(header_row)) if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError( 'expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) return points
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i+1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError( 'expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # find the set of indices of duplicate points dup_indices = get_dup_indices(points, fs.radius) # get the data rows with duplicate indices removed new_rows = [row for i, row in enumerate(data_rows) if i not in dup_indices] # construct the new table out = StringIO() print >> out, '\t'.join(header_row) print >> out, '\n'.join('\t'.join(row) for row in new_rows) return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i + 1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: msg_a = 'expected the axis column %s ' % h msg_b = 'to be numeric' raise ValueError(msg_a + msg_b) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # precompute some stuff allmeandist = kmeans.get_allmeandist(points) nrestarts = 10 nseconds = 2 tm = time.time() n = len(points) wgss_list = [] # neg because both items in the pair are used for sorting neg_calinski_k_pairs = [] # look for the best calinski index in a small amount of time k = 2 while True: codebook, distortion = cluster.vq.kmeans(points, k, iter=nrestarts, thresh=1e-9) sqdists = kmeans.get_point_center_sqdists(points, codebook) labels = kmeans.get_labels_without_cluster_removal(sqdists) wgss = kmeans.get_wcss(sqdists, labels) bgss = allmeandist - wgss calinski = kmeans.get_calinski_index(bgss, wgss, k, n) k_unique = len(set(labels)) neg_calinski_k_pairs.append((-calinski, k_unique)) wgss_list.append(wgss) if time.time() - tm > nseconds: break if k == n - 1: break k += 1 max_k = k best_neg_calinski, best_k = min(neg_calinski_k_pairs) best_calinski = -best_neg_calinski # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k print >> out, 'searched 2 <= k <= %d clusters' % max_k print >> out, '%.2f seconds' % (time.time() - tm) if fs.verbose: print >> out print >> out, '(k_unique, wgss, calinski):' for wgss, neg_calinski_k_pair in zip(wgss_list, neg_calinski_k_pairs): neg_calinski, k_unique = neg_calinski_k_pair calinski = -neg_calinski row = [k_unique, wgss, calinski] print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i+1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: msg_a = 'expected the axis column %s ' % h msg_b = 'to be numeric' raise ValueError(msg_a + msg_b) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # precompute some stuff allmeandist = kmeans.get_allmeandist(points) nrestarts = 10 nseconds = 2 tm = time.time() n = len(points) wgss_list = [] # neg because both items in the pair are used for sorting neg_calinski_k_pairs = [] # look for the best calinski index in a small amount of time k = 2 while True: codebook, distortion = cluster.vq.kmeans( points, k, iter=nrestarts, thresh=1e-9) sqdists = kmeans.get_point_center_sqdists(points, codebook) labels = kmeans.get_labels_without_cluster_removal(sqdists) wgss = kmeans.get_wcss(sqdists, labels) bgss = allmeandist - wgss calinski = kmeans.get_calinski_index(bgss, wgss, k, n) k_unique = len(set(labels)) neg_calinski_k_pairs.append((-calinski, k_unique)) wgss_list.append(wgss) if time.time() - tm > nseconds: break if k == n-1: break k += 1 max_k = k best_neg_calinski, best_k = min(neg_calinski_k_pairs) best_calinski = -best_neg_calinski # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k print >> out, 'searched 2 <= k <= %d clusters' % max_k print >> out, '%.2f seconds' % (time.time() - tm) if fs.verbose: print >> out print >> out, '(k_unique, wgss, calinski):' for wgss, neg_calinski_k_pair in zip(wgss_list, neg_calinski_k_pairs): neg_calinski, k_unique = neg_calinski_k_pair calinski = -neg_calinski row = [k_unique, wgss, calinski] print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i + 1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError('expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # do the clustering while computing the wgss at each merge cluster_counts = [] wgss_values = [] allmeandist = kmeans.get_allmeandist(points) cluster_map = agglom.get_initial_cluster_map(points) w_ssd_map = agglom.get_initial_w_ssd_map(points) b_ssd_map = agglom.get_initial_b_ssd_map(points) q = agglom.get_initial_queue(b_ssd_map) while len(cluster_map) > 2: # do an agglomeration step pair = agglom.get_pair_fast(cluster_map, q) agglom.merge_fast(cluster_map, w_ssd_map, b_ssd_map, q, pair) # compute the within group sum of squares indices = cluster_map.keys() wgss = sum(w_ssd_map[i] / float(len(cluster_map[i])) for i in indices) # compute the between group sum of squares bgss = allmeandist - wgss # append to the lists cluster_counts.append(len(cluster_map)) wgss_values.append(wgss) # compute the log wgss values wlogs = np.log(wgss_values) # reverse the log values so that they are by increasing cluster size wlogs = list(reversed(wlogs)) # sample from the null distribution extents = np.max(points, axis=0) - np.min(points, axis=0) nclusters_list, expectations, thresholds = do_sampling( extents, len(points), fs.nsamples) # get the gaps gaps = np.array(expectations) - wlogs # Get the best cluster count according to the gap statistic. best_i = None criteria = [] for i, ip1 in iterutils.pairwise(range(len(nclusters_list))): k, kp1 = nclusters_list[i], nclusters_list[ip1] criterion = gaps[i] - gaps[ip1] + thresholds[ip1] criteria.append(criterion) if criterion > 0: if best_i is None: best_i = i best_k = nclusters_list[best_i] # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k if fs.verbose: print >> out print >> out, '(k, expected, observed, gap, threshold, criterion):' n = len(nclusters_list) for i, k in enumerate(nclusters_list): row = [k, expectations[i], wlogs[i], gaps[i], thresholds[i]] if i < n - 1: row += [criteria[i]] else: row += ['-'] print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i+1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError( 'expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # do the clustering while computing the calinski index at each merge cluster_counts = [] wgss_values = [] neg_calinskis = [] allmeandist = kmeans.get_allmeandist(points) cluster_map = agglom.get_initial_cluster_map(points) w_ssd_map = agglom.get_initial_w_ssd_map(points) b_ssd_map = agglom.get_initial_b_ssd_map(points) q = agglom.get_initial_queue(b_ssd_map) while len(cluster_map) > 2: # do an agglomeration step pair = agglom.get_pair_fast(cluster_map, q) agglom.merge_fast(cluster_map, w_ssd_map, b_ssd_map, q, pair) # compute the within group sum of squares indices = cluster_map.keys() wgss = sum(w_ssd_map[i] / float(len(cluster_map[i])) for i in indices) # compute the between group sum of squares bgss = allmeandist - wgss # get the calinksi index n = len(points) k = len(cluster_map) numerator = bgss / float(k - 1) denominator = wgss / float(n - k) calinski = numerator / denominator # append to the lists cluster_counts.append(k) wgss_values.append(wgss) neg_calinskis.append(-calinski) # Get the best cluster count according to the calinski index. # Do this trickery with negs so that it breaks ties # using the smallest number of clusters. neg_calinksi, best_k = min(zip(neg_calinskis, cluster_counts)) # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k if fs.verbose: print >> out print >> out, '(k, wgss, calinski):' for k, wgss, neg_calinski in zip( cluster_counts, wgss_values, neg_calinskis): row = (k, wgss, -neg_calinski) print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i+1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError( 'expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # do the clustering while computing the wgss at each merge cluster_counts = [] wgss_values = [] allmeandist = kmeans.get_allmeandist(points) cluster_map = agglom.get_initial_cluster_map(points) w_ssd_map = agglom.get_initial_w_ssd_map(points) b_ssd_map = agglom.get_initial_b_ssd_map(points) q = agglom.get_initial_queue(b_ssd_map) while len(cluster_map) > 2: # do an agglomeration step pair = agglom.get_pair_fast(cluster_map, q) agglom.merge_fast(cluster_map, w_ssd_map, b_ssd_map, q, pair) # compute the within group sum of squares indices = cluster_map.keys() wgss = sum(w_ssd_map[i] / float(len(cluster_map[i])) for i in indices) # compute the between group sum of squares bgss = allmeandist - wgss # append to the lists cluster_counts.append(len(cluster_map)) wgss_values.append(wgss) # compute the log wgss values wlogs = np.log(wgss_values) # reverse the log values so that they are by increasing cluster size wlogs = list(reversed(wlogs)) # sample from the null distribution extents = np.max(points, axis=0) - np.min(points, axis=0) nclusters_list, expectations, thresholds = do_sampling( extents, len(points), fs.nsamples) # get the gaps gaps = np.array(expectations) - wlogs # Get the best cluster count according to the gap statistic. best_i = None criteria = [] for i, ip1 in iterutils.pairwise(range(len(nclusters_list))): k, kp1 = nclusters_list[i], nclusters_list[ip1] criterion = gaps[i] - gaps[ip1] + thresholds[ip1] criteria.append(criterion) if criterion > 0: if best_i is None: best_i = i best_k = nclusters_list[best_i] # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k if fs.verbose: print >> out print >> out, '(k, expected, observed, gap, threshold, criterion):' n = len(nclusters_list) for i, k in enumerate(nclusters_list): row = [k, expectations[i], wlogs[i], gaps[i], thresholds[i]] if i < n-1: row += [criteria[i]] else: row += ['-'] print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()
def get_response_content(fs): # read the table rtable = RUtil.RTable(fs.table.splitlines()) header_row = rtable.headers data_rows = rtable.data Carbone.validate_headers(header_row) # get the numpy array of conformant points h_to_i = dict((h, i + 1) for i, h in enumerate(header_row)) axis_headers = fs.axes if not axis_headers: raise ValueError('no Euclidean axes were provided') axis_set = set(axis_headers) header_set = set(header_row) bad_axes = axis_set - header_set if bad_axes: raise ValueError('invalid axes: ' + ', '.join(bad_axes)) axis_lists = [] for h in axis_headers: index = h_to_i[h] try: axis_list = Carbone.get_numeric_column(data_rows, index) except Carbone.NumericError: raise ValueError('expected the axis column %s ' 'to be numeric' % h) axis_lists.append(axis_list) points = np.array(zip(*axis_lists)) # do the clustering while computing the calinski index at each merge cluster_counts = [] wgss_values = [] neg_calinskis = [] allmeandist = kmeans.get_allmeandist(points) cluster_map = agglom.get_initial_cluster_map(points) w_ssd_map = agglom.get_initial_w_ssd_map(points) b_ssd_map = agglom.get_initial_b_ssd_map(points) q = agglom.get_initial_queue(b_ssd_map) while len(cluster_map) > 2: # do an agglomeration step pair = agglom.get_pair_fast(cluster_map, q) agglom.merge_fast(cluster_map, w_ssd_map, b_ssd_map, q, pair) # compute the within group sum of squares indices = cluster_map.keys() wgss = sum(w_ssd_map[i] / float(len(cluster_map[i])) for i in indices) # compute the between group sum of squares bgss = allmeandist - wgss # get the calinksi index n = len(points) k = len(cluster_map) numerator = bgss / float(k - 1) denominator = wgss / float(n - k) calinski = numerator / denominator # append to the lists cluster_counts.append(k) wgss_values.append(wgss) neg_calinskis.append(-calinski) # Get the best cluster count according to the calinski index. # Do this trickery with negs so that it breaks ties # using the smallest number of clusters. neg_calinksi, best_k = min(zip(neg_calinskis, cluster_counts)) # create the response out = StringIO() print >> out, 'best cluster count: k = %d' % best_k if fs.verbose: print >> out print >> out, '(k, wgss, calinski):' for k, wgss, neg_calinski in zip(cluster_counts, wgss_values, neg_calinskis): row = (k, wgss, -neg_calinski) print >> out, '\t'.join(str(x) for x in row) # return the response return out.getvalue()