def test_data_named_gene(self):
var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["RER1"]
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1)
self.assertEqual(data["col_idx"], [4])
filter_ = {
"filter": {
"var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["SPEN", "TYMP", "PRMT2"]
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
def test_annotation_fields(self):
fbs = self.data.annotation_to_fbs_matrix("obs",
["n_genes", "n_counts"])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
def test_filtered_data_frame(self):
filter_ = {
"filter": {
"var": {
"annotation_value": [{
"name": "n_cells",
"min": 100
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040)
filter_ = {
"filter": {
"obs": {
"annotation_value": [{
"name": "n_counts",
"min": 3000
}]
}
}
}
with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_frame(self):
fbs = self.data.data_frame_to_fbs_matrix(None, "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1838)
with self.assertRaises(ValueError):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_annotation_var(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 10)
with self.assertRaises(ValueError):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_layout(self):
fbs = self.data.layout_to_fbs_matrix()
layout = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(layout["n_cols"], 2)
self.assertEqual(layout["n_rows"], 2638)
X = layout["columns"][0]
self.assertTrue((X >= 0).all() and (X <= 1).all())
Y = layout["columns"][1]
self.assertTrue((Y >= 0).all() and (Y <= 1).all())
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("obs"))
self.assertEqual(
annotations["col_idx"],
["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("var"))
self.assertEqual(annotations["col_idx"],
["name", "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.annotation(None, "obs"))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.annotation(None, "var"))
def test_data_put_single_var(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_annotations(self):
fbs = self.data.annotation_to_fbs_matrix("obs")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[
obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain"
],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"],
[var_index_col_name, "n_cells"])
def test_dataframe(self):
data_frame_var = decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "var"))
self.assertIsNotNone(data_frame_var)
self.assertEqual(data_frame_var["n_rows"], 100)
self.assertEqual(data_frame_var["n_cols"], 100)
self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.data_frame(None, "obs"))
with pytest.raises(JSONEncodingValueError):
json.loads(self.data.data_frame(None, "var"))
def test_put_get_roundtrip(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A':
pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
# put
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
fbsAll = self.data.annotation_to_fbs_matrix("obs")
schema = self.data.get_schema()
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain", "cat_A", "cat_B"
])
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')],
['label_A' for l in range(0, n_rows)])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')],
['label_B' for l in range(0, n_rows)])
# verify the schema was updated
all_col_schema = {
c["name"]: c
for c in schema["annotations"]["obs"]["columns"]
}
self.assertEqual(
all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(
all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
def test_dataframe(self):
data_frame_var = decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "var"))
self.assertIsNotNone(data_frame_var)
self.assertEqual(data_frame_var["n_rows"], 100)
self.assertEqual(data_frame_var["n_cols"], 100)
self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError):
filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_put_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'].tolist(), [])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
def test_filter_complex(self):
filter_ = {
"filter": {
"var": {
"annotation_value": [{
"name": "n_cells",
"min": 10
}],
"index": [1, 99, [200, 300]]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 91)
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 8)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'], [
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
def test_filter_idx(self):
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 102)
def test_dataframe_obs_not_implemented(self):
with self.assertRaises(ValueError) as cm:
decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "obs"))
self.assertIsNotNone(cm.exception)
