def test_export_fcs_as_raw(self):
# This test uses a file where the preprocessing makes the orig & raw events different.
# The purpose here is to verify that importing the exported file has the same raw events
# as the original file's raw events.
fcs_file_path = "examples/data/8_color_data_set/fcs_files/101_DEN084Y5_15_E01_008_clean.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
sample.export("test_fcs_export.fcs",
source='raw',
directory="examples")
exported_fcs_file = "examples/test_fcs_export.fcs"
exported_sample = Sample(fcs_path_or_data=exported_fcs_file)
os.unlink(exported_fcs_file)
self.assertIsInstance(exported_sample, Sample)
# When the events are exported, they are saved as single precision (32-bit). We'll test the
# arrays with the original sample data converted to 32-bit float. The original sample data
# was also originally in 32-bit. Comparing both in single precision is then the most
# "correct" thing to do here.
np.testing.assert_array_equal(
sample._raw_events.astype(np.float32),
exported_sample._raw_events.astype(np.float32))
def test_export_fcs_as_raw_with_gain(self):
# This test uses a file where the preprocessing makes the orig & raw events different.
# File data1.fcs has 2 channels that specify a gain value other than 1.0.
# The purpose here is to verify that importing the exported file has the same raw events
# as the original file's raw events.
sample = Sample(fcs_path_or_data=data1_fcs_path,
cache_original_events=True)
sample.export("test_fcs_export.fcs",
source='raw',
directory="examples")
exported_fcs_file = "examples/test_fcs_export.fcs"
exported_sample = Sample(fcs_path_or_data=exported_fcs_file)
os.unlink(exported_fcs_file)
self.assertIsInstance(exported_sample, Sample)
# When the events are exported, they are saved as single precision (32-bit). We'll test the
# arrays with the original sample data converted to 32-bit float. The original sample data
# was also originally in 32-bit. Comparing both in single precision is then the most
# "correct" thing to do here.
np.testing.assert_array_equal(
sample._raw_events.astype(np.float32),
exported_sample._raw_events.astype(np.float32))
def test_get_comp_events_if_no_comp(self):
fcs_file_path = "examples/test_comp_example.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
comp_events = sample.get_comp_events()
self.assertIsNone(comp_events)
def test_get_transformed_events_if_no_xform(self):
fcs_file_path = "examples/test_comp_example.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
xform_events = sample.get_transformed_events()
self.assertIsNone(xform_events)
def test_get_subsampled_orig_events(self):
sample = Sample(data1_fcs_path,
cache_original_events=True,
subsample=500)
events = sample.get_events(source='orig', subsample=True)
self.assertEqual(events.shape[0], 500)
def test_get_metadata(self):
"""Test Sample method get_metadata"""
fcs_file_path = "examples/data/test_data_2d_01.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
meta = sample.get_metadata()
self.assertEqual(len(meta), 20)
self.assertEqual(meta['p1n'], 'channel_A')
def test_get_channel_data_comp():
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
data_idx_6 = sample.get_channel_data(6, source='comp')
np.testing.assert_equal(sample._comp_events[:, 6], data_idx_6)
def test_clearing_comp_events(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = "examples/comp_complete_example.csv"
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
sample.apply_compensation(None)
self.assertIsNone(sample._comp_events)
def test_get_channel_data_xform():
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
sample.apply_transform(xform_logicle)
data_idx_6 = sample.get_channel_data(6, source='xform')
np.testing.assert_equal(sample._transformed_events[:, 6], data_idx_6)
def test_get_events_as_data_frame_comp(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = "examples/comp_complete_example.csv"
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
df = sample.as_dataframe(source='comp')
self.assertIsInstance(df, pd.DataFrame)
np.testing.assert_equal(df.values, sample.get_comp_events())
def test_load_from_pandas_multi_index(self):
sample_orig = Sample("examples/data/100715.fcs",
cache_original_events=True)
pnn_orig = sample_orig.pnn_labels
pns_orig = sample_orig.pns_labels
df = sample_orig.as_dataframe(source='orig')
sample_new = Sample(df)
pnn_new = sample_new.pnn_labels
pns_new = sample_new.pns_labels
self.assertListEqual(pnn_orig, pnn_new)
self.assertListEqual(pns_orig, pns_new)
def test_load_from_fcs_file_path(self):
"""Test creating Sample object from an FCS file path"""
fcs_file_path = "examples/test_data_2d_01.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
self.assertIsInstance(sample, Sample)
def test_load_from_pathlib(self):
"""Test creating Sample object from a pathlib Path object"""
fcs_file_path = "examples/test_data_2d_01.fcs"
path = Path(fcs_file_path)
sample = Sample(fcs_path_or_data=path)
self.assertIsInstance(sample, Sample)
def test_load_from_io_base(self):
"""Test creating Sample object from a IOBase object"""
fcs_file_path = "examples/data/test_data_2d_01.fcs"
f = open(fcs_file_path, 'rb')
sample = Sample(fcs_path_or_data=f)
f.close()
self.assertIsInstance(sample, Sample)
def test_comp_matrix_from_pathlib_path(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
self.assertIsNotNone(sample._comp_events)
def test_get_subsampled_orig_events_not_cached(self):
sample = Sample(data1_fcs_path,
cache_original_events=False,
subsample=500)
self.assertRaises(ValueError,
sample.get_events,
source='orig',
subsample=True)
def load_data(sample: fk.Sample):
channels = range(SKIP, len(sample.channels))
channel_data = []
for channel in channels:
channel_data.append(
np.array(sample.get_channel_data(channel, source="raw")))
return np.array(channel_data).T
def test_load_from_pathlib(self):
"""Test creating Sample object from a pathlib Path object"""
fcs_file_path = "examples/test_data_2d_01.fcs"
path = Path(fcs_file_path)
sample = Sample(fcs_path_or_data=path,
filter_negative_scatter=False,
filter_anomalous_events=False)
self.assertIsInstance(sample, Sample)
def test_comp_matrix_from_pathlib_path(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path,
filter_negative_scatter=False,
filter_anomalous_events=False)
self.assertIsNotNone(sample._comp_events)
def test_load_from_flowio_flowdata_object(self):
"""Test creating Sample object from an FCS file path"""
fcs_file_path = "examples/data/test_data_2d_01.fcs"
flow_data = flowio.FlowData(fcs_file_path)
self.assertIsInstance(flow_data, flowio.FlowData)
sample = Sample(flow_data)
self.assertIsInstance(sample, Sample)
def test_export_fcs_as_orig_with_pne_log_raises_error(self):
# This test uses a file where some PnE values specify log scale
# FlowIO does not currently support creating FCS files with log scale PnE values
# Test that the Sample class raises a NotImplementedError for this case.
sample = Sample(fcs_path_or_data=data1_fcs_path,
cache_original_events=True)
self.assertRaises(NotImplementedError,
sample.export,
"test_fcs_export.fcs",
source='orig',
directory="examples")
def test_create_csv(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
sample.export("test_fcs_export.csv",
source='comp',
directory="examples")
exported_csv_file = "examples/test_fcs_export.csv"
exported_df = pd.read_csv(exported_csv_file)
exported_sample = Sample(exported_df)
os.unlink(exported_csv_file)
self.assertIsInstance(exported_sample, Sample)
# TODO: Need to investigate why the exported comp data isn't exactly equal
np.testing.assert_almost_equal(sample._comp_events[:, :],
exported_sample._raw_events[:, :],
decimal=3)
def test_create_fcs(self):
fcs_file_path = "examples/test_comp_example.fcs"
comp_file_path = Path("examples/comp_complete_example.csv")
sample = Sample(fcs_path_or_data=fcs_file_path,
compensation=comp_file_path)
sample.export("test_fcs_export.fcs",
source='comp',
directory="examples")
exported_fcs_file = "examples/test_fcs_export.fcs"
exported_sample = Sample(fcs_path_or_data=exported_fcs_file)
os.unlink(exported_fcs_file)
self.assertIsInstance(exported_sample, Sample)
# TODO: Excluding time channel here, as the difference was nearly 0.01. Need to investigate why the
# exported comp data isn't exactly equal
np.testing.assert_almost_equal(sample._comp_events[:, :-1],
exported_sample._raw_events[:, :-1],
decimal=3)
def test_load_from_numpy_array(self):
npy_file_path = "examples/test_comp_example.npy"
channels = [
'FSC-A', 'FSC-W', 'SSC-A', 'Ax488-A', 'PE-A', 'PE-TR-A',
'PerCP-Cy55-A', 'PE-Cy7-A', 'Ax647-A', 'Ax700-A', 'Ax750-A',
'PacBlu-A', 'Qdot525-A', 'PacOrange-A', 'Qdot605-A', 'Qdot655-A',
'Qdot705-A', 'Time'
]
npy_data = np.fromfile(npy_file_path)
sample = Sample(npy_data, channel_labels=channels)
self.assertIsInstance(sample, Sample)
def get_samples_from_paths(sample_paths):
if multi_proc:
if len(sample_paths) < mp.cpu_count():
proc_count = len(sample_paths)
else:
proc_count = mp.cpu_count() - 1 # leave a CPU free just to be nice
pool = mp.Pool(processes=proc_count)
samples = pool.map(Sample, sample_paths)
else:
samples = []
for path in sample_paths:
samples.append(Sample(path))
return samples
def test_filter_anomalous_events(self):
# there are 2 negative SSC-A events in this file (of 65016 total events)
fcs_file_path = "examples/100715.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
sample.subsample_events(50000)
sample.filter_anomalous_events(reapply_subsample=False)
# using the default seed, the 2 negative events are in the subsample
common_idx = np.intersect1d(sample.subsample_indices,
sample.anomalous_indices)
self.assertGreater(common_idx.shape[0], 0)
sample.filter_anomalous_events(reapply_subsample=True)
common_idx = np.intersect1d(sample.subsample_indices,
sample.anomalous_indices)
self.assertEqual(common_idx.shape[0], 0)
self.assertGreater(sample.anomalous_indices.shape[0], 0)
def test_export_as_fcs(self):
sample = test_comp_sample
sample.export("test_fcs_export.fcs",
source='comp',
directory="examples")
exported_fcs_file = "examples/test_fcs_export.fcs"
exported_sample = Sample(fcs_path_or_data=exported_fcs_file)
os.unlink(exported_fcs_file)
self.assertIsInstance(exported_sample, Sample)
# When the comp events are exported, they are saved as single precision (32-bit). We'll test the
# arrays with the original sample comp data converted to 32-bit float. The original sample data
# was also originally in 32-bit but the compensation calculation results in 64-bit data. Comparing
# both in single precision is then the most "correct" thing to do here.
np.testing.assert_array_equal(
sample._comp_events.astype(np.float32),
exported_sample._raw_events.astype(np.float32))
def test_export_exclude_negative_scatter(self):
# there are 2 negative SSC-A events in this file (of 65016 total events)
fcs_file_path = "examples/data/100715.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
sample.filter_negative_scatter()
neg_scatter_count = len(sample.negative_scatter_indices)
exported_fcs_file = "examples/test_fcs_export.fcs"
sample.export(exported_fcs_file,
source='raw',
exclude_neg_scatter=True)
exported_sample = Sample(exported_fcs_file)
os.unlink(exported_fcs_file)
orig_event_count = sample.event_count
exp_event_count = exported_sample.event_count
self.assertEqual(exp_event_count, orig_event_count - neg_scatter_count)
def test_filter_negative_scatter(self):
# there are 2 negative SSC-A events in this file (of 65016 total events)
fcs_file_path = "examples/data/100715.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path, subsample=50000)
sample.filter_negative_scatter(reapply_subsample=False)
# using the default seed, the 2 negative events are in the subsample
common_idx = np.intersect1d(sample.subsample_indices,
sample.negative_scatter_indices)
self.assertEqual(len(common_idx), 2)
sample.filter_negative_scatter(reapply_subsample=True)
common_idx = np.intersect1d(sample.subsample_indices,
sample.negative_scatter_indices)
self.assertEqual(len(common_idx), 0)
self.assertEqual(sample.negative_scatter_indices.shape[0], 2)
"""
Unit tests for Sample class
"""
import unittest
import sys
import os
from pathlib import Path
import numpy as np
import pandas as pd
sys.path.append(os.path.abspath('..'))
from flowkit import Sample, transforms
data1_fcs_path = 'examples/gate_ref/data1.fcs'
data1_sample = Sample(data1_fcs_path)
xform_logicle = transforms.LogicleTransform('logicle',
param_t=10000,
param_w=0.5,
param_m=4.5,
param_a=0)
class LoadSampleTestCase(unittest.TestCase):
"""Tests for loading FCS files as Sample objects"""
def test_load_from_fcs_file_path(self):
"""Test creating Sample object from an FCS file path"""
fcs_file_path = "examples/test_data_2d_01.fcs"
sample = Sample(fcs_path_or_data=fcs_file_path)
