def layout_to_fbs_matrix(self, fields):
"""
return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
* returns only first two dimensions, with name {ename}_0 and {ename}_1,
where {ename} is the embedding name.
* client assumes each will be individually centered & scaled (isotropically)
to a [0, 1] range.
* does not support filtering
"""
embeddings = self.get_embedding_names(
) if fields is None or len(fields) == 0 else fields
layout_data = []
with ServerTiming.time("layout.query"):
for ename in embeddings:
embedding = self.get_embedding_array(ename, 2)
normalized_layout = DataAdaptor.normalize_embedding(embedding)
layout_data.append(
pd.DataFrame(normalized_layout,
columns=[f"{ename}_0", f"{ename}_1"]))
with ServerTiming.time("layout.encode"):
if layout_data:
df = pd.concat(layout_data, axis=1, copy=False)
else:
df = pd.DataFrame()
fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
return fbs
def test_type_conversion_consistency(self):
self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types))
self.assertEqual(self.dataframe.shape[1],
len(self.expected_schema_hints))
buf = encode_matrix_fbs(matrix=self.dataframe,
col_idx=self.dataframe.columns)
encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(
self.dataframe)
# check schema hints
# print(schema_hints)
# print(self.expected_schema_hints)
self.assertEqual(schema_hints, self.expected_schema_hints)
# inspect the FBS types
matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0)
columns_length = matrix.ColumnsLength()
self.assertEqual(columns_length, self.dataframe.shape[1])
self.assertEqual(
matrix.ColIndexType(),
fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray)
col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray()
col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos)
col_index_labels = json.loads(
col_labels_arr.DataAsNumpy().tobytes().decode("utf-8"))
self.assertEqual(len(col_index_labels), self.dataframe.shape[1])
for col_idx in range(0, columns_length):
col_label = col_index_labels[col_idx]
col = matrix.Columns(col_idx)
col_type = col.UType()
self.assertEqual(self.expected_fbs_types[col_label], col_type)
def data_frame_to_fbs_matrix(self, filter, axis):
"""
Retrieves data 'X' and returns in a flatbuffer Matrix.
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: flatbuffer Matrix
Caveats:
* currently only supports access on VAR axis
* currently only supports filtering on VAR axis
"""
if axis != Axis.VAR:
raise ValueError("Only VAR dimension access is supported")
try:
obs_selector, var_selector = self._filter_to_mask(filter)
except (KeyError, IndexError, TypeError, AttributeError):
raise FilterError("Error parsing filter")
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
num_columns = self.get_shape(
)[1] if var_selector is None else np.count_nonzero(var_selector)
if self.server_config.exceeds_limit("column_request_max", num_columns):
raise ExceedsLimitError(
"Requested dataframe columns exceed column request limit")
X = self.get_X_array(obs_selector, var_selector)
col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
if axis == Axis.OBS:
if labels is not None and not labels.empty:
df = self.data.obs.join(labels,
self.parameters.get("obs_names"))
else:
df = self.data.obs
else:
df = self.data.var
if fields is not None and len(fields) > 0:
df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns)
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
"a":
np.zeros((10, ), dtype=np.float32),
"b":
np.ones((10, ), dtype=np.int64),
"c":
np.array([i for i in range(0, 10)], dtype=np.uint16),
"d":
pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"],
dtype="category"),
})
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32),
(np.ndarray, np.int32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_boundary(self):
"""test various boundary checks"""
# row indexing is unsupported
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
# matrix must be 2D
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.ones((10, )))
def summarize_var(self, method, filter, query_hash):
if method != "mean":
raise UnsupportedSummaryMethod("Unknown gene set summary method.")
obs_selector, var_selector = self._filter_to_mask(filter)
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
# if no filter, just return zeros. We don't have a use case
# for summarizing the entire X without a filter, and it would
# potentially be quite compute / memory intensive.
if var_selector is None or np.count_nonzero(var_selector) == 0:
mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
else:
X = self.get_X_array(obs_selector, var_selector)
if sparse.issparse(X):
mean = X.mean(axis=1).A
else:
mean = X.mean(axis=1, keepdims=True)
col_idx = pd.Index([query_hash])
return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
"a":
np.zeros((10, ), dtype=np.float32),
"b":
np.ones((10, ), dtype=np.int64),
"c":
np.array([i for i in range(0, 10)], dtype=np.uint16),
"d":
pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"],
dtype="category"),
})
dfDst = decode_matrix_fbs(
encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns:
self.assertTrue(c in dfDst.columns)
if isinstance(dfSrc[c], pd.Series):
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
else:
self.assertEqual(dfSrc[c], dfDst[c])
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32),
(np.ndarray, np.int32))
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32),
(np.ndarray, np.float32))
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def make_fbs(data):
df = pd.DataFrame(data)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)