def layout_to_fbs_matrix(self, fields): """ return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding. * returns only first two dimensions, with name {ename}_0 and {ename}_1, where {ename} is the embedding name. * client assumes each will be individually centered & scaled (isotropically) to a [0, 1] range. * does not support filtering """ embeddings = self.get_embedding_names( ) if fields is None or len(fields) == 0 else fields layout_data = [] with ServerTiming.time("layout.query"): for ename in embeddings: embedding = self.get_embedding_array(ename, 2) normalized_layout = DataAdaptor.normalize_embedding(embedding) layout_data.append( pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"])) with ServerTiming.time("layout.encode"): if layout_data: df = pd.concat(layout_data, axis=1, copy=False) else: df = pd.DataFrame() fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) return fbs
def data_frame_to_fbs_matrix(self, filter, axis): """ Retrieves data 'X' and returns in a flatbuffer Matrix. :param filter: filter: dictionary with filter params :param axis: string obs or var :return: flatbuffer Matrix Caveats: * currently only supports access on VAR axis * currently only supports filtering on VAR axis """ if axis != Axis.VAR: raise ValueError("Only VAR dimension access is supported") try: obs_selector, var_selector = self._filter_to_mask(filter) except (KeyError, IndexError, TypeError, AttributeError): raise FilterError("Error parsing filter") if obs_selector is not None: raise FilterError("filtering on obs unsupported") num_columns = self.get_shape( )[1] if var_selector is None else np.count_nonzero(var_selector) if self.server_config.exceeds_limit("column_request_max", num_columns): raise ExceedsLimitError( "Requested dataframe columns exceed column request limit") X = self.get_X_array(obs_selector, var_selector) col_idx = np.nonzero([] if var_selector is None else var_selector)[0] return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
def annotation_to_fbs_matrix(self, axis, fields=None, labels=None): if axis == Axis.OBS: if labels is not None and not labels.empty: df = self.data.obs.join(labels, self.parameters.get("obs_names")) else: df = self.data.obs else: df = self.data.var if fields is not None and len(fields) > 0: df = df[fields] return encode_matrix_fbs(df, col_idx=df.columns)
def test_encode_DataFrame(self): df = pd.DataFrame( data={ "a": np.zeros((10, ), dtype=np.float32), "b": np.ones((10, ), dtype=np.int64), "c": np.array([i for i in range(0, 10)], dtype=np.uint16), "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), }) expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_boundary(self): """ test various boundary checks """ # row indexing is unsupported with self.assertRaises(ValueError): encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[]) # matrix must be 2D with self.assertRaises(ValueError): encode_matrix_fbs(matrix=np.zeros((3, 2, 1))) with self.assertRaises(ValueError): encode_matrix_fbs(matrix=np.ones((10, )))
def test_roundtrip(self): dfSrc = pd.DataFrame( data={ "a": np.zeros((10, ), dtype=np.float32), "b": np.ones((10, ), dtype=np.int64), "c": np.array([i for i in range(0, 10)], dtype=np.uint16), "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), }) dfDst = decode_matrix_fbs( encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) for c in dfSrc.columns: self.assertTrue(c in dfDst.columns) if isinstance(dfSrc[c], pd.Series): self.assertTrue(np.all(dfSrc[c] == dfDst[c])) else: self.assertEqual(dfSrc[c], dfDst[c])
def test_encode_sparse(self): csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32)) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_encode_ndarray(self): arr = np.zeros((3, 2), dtype=np.float32) expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32)) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) self.fbs_checks(fbs, (3, 2), expected_types, None)
def make_fbs(data): df = pd.DataFrame(data) return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)