def verify_results(self, tempdir: Path):
intensities = IntensityTable.open_netcdf(
os.fspath(tempdir / "decoded_spots.nc"))
genes, counts = np.unique(intensities.coords[Features.TARGET],
return_counts=True)
gene_counts = pd.Series(counts, genes)
assert gene_counts['ACTB'] == 9
assert gene_counts['GAPDH'] == 9
def _load_data(fov_data, exp):
"""
Load a field of view dataset
Parameters
----------
fov_data : pd.DataFrame
Table of file locations for the spot results and mask label image
corresponding to the dataset to be viewed
exp : Experiment
Experiment file corresponding to the data analysis
Returns
-------
im_max_proj : np.ndarray
Image used for the spot detection
points : List[Dict]
List of dictionaries containing the points and target names for the
points to be rendered
mask_im : np.ndarray
Label image for the segmentation mask
"""
# Get the RNAscope images
im = exp[fov_data.fov_name].get_image("primary")
im_max_proj = np.squeeze(im.max_proj(Axes.ZPLANE).xarray.values)
# Get the spots
spots_file = fov_data.spot_file
it = IntensityTable.open_netcdf(spots_file)
points = _get_points(it, exp)
# Get the segmentation mask
mask_file = fov_data.mask_file
mask_im = io.imread(mask_file)
return im_max_proj, points, mask_im
def test_dartfish_pipeline_cropped_data(tmpdir):
# set random seed to errors provoked by optimization functions
np.random.seed(777)
dartfish = __import__('DARTFISH')
primary_image = dartfish.imgs
expected_primary_image = np.array(
[[1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 4.57770657e-05, 0.00000000e+00,
0.00000000e+00, 3.05180438e-05],
[0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00],
[0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
3.05180438e-05, 0.00000000e+00],
[0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00,
0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 1.52590219e-05,
0.00000000e+00, 0.00000000e+00],
[0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 3.05180438e-05, 0.00000000e+00,
1.52590219e-05, 0.00000000e+00],
[0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00,
3.05180438e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 1.52590219e-05],
[0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00],
[0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00,
1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00],
[1.52590219e-05, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00],
[0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,
0.00000000e+00, 1.52590219e-05, 1.52590219e-05, 0.00000000e+00,
0.00000000e+00, 0.00000000e+00]],
dtype=np.float32
)
assert primary_image.xarray.dtype == np.float32
assert np.allclose(
primary_image.xarray[0, 0, 0, 50:60, 60:70],
expected_primary_image
)
normalized_image = dartfish.norm_imgs
expected_normalized_image = np.array(
[[0.01960784, 0., 0., 0., 0.,
0., 0.05882353, 0., 0., 0.03921569],
[0., 0., 0., 0., 0.01960784,
0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.01960784,
0., 0., 0., 0.03921569, 0.],
[0., 0., 0.01960784, 0., 0.,
0.01960784, 0., 0.01960784, 0., 0.],
[0., 0., 0., 0., 0.,
0., 0.03921569, 0., 0.01960784, 0.],
[0., 0., 0.01960784, 0., 0.03921569,
0., 0., 0., 0., 0.01960784],
[0., 0., 0., 0., 0.,
0.01960784, 0., 0., 0., 0.],
[0., 0.01960784, 0., 0., 0.01960784,
0., 0., 0., 0., 0.],
[0.01960784, 0.01960784, 0., 0., 0.,
0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.,
0.01960784, 0.01960784, 0., 0., 0.]],
dtype=np.float32,
)
assert normalized_image.xarray.dtype == np.float32
assert np.allclose(
normalized_image.xarray[0, 0, 0, 50:60, 60:70],
expected_normalized_image
)
zero_norm_stack = dartfish.filtered_imgs
expected_zero_normalized_image = np.array(
[[0.01960784, 0., 0., 0., 0.,
0., 0.05882353, 0., 0., 0.03921569],
[0., 0., 0., 0., 0.01960784,
0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.01960784,
0., 0., 0., 0., 0.],
[0., 0., 0.01960784, 0., 0.,
0., 0., 0.01960784, 0., 0.],
[0., 0., 0., 0., 0.,
0., 0.03921569, 0., 0.01960784, 0.],
[0., 0., 0.01960784, 0., 0.,
0., 0., 0., 0., 0.01960784],
[0., 0., 0., 0., 0.,
0.01960784, 0., 0., 0., 0.],
[0., 0.01960784, 0., 0., 0.,
0., 0., 0., 0., 0.],
[0.01960784, 0., 0., 0., 0.,
0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.,
0., 0.01960784, 0., 0., 0.]],
dtype=np.float32
)
assert np.allclose(
expected_zero_normalized_image,
zero_norm_stack.xarray[0, 0, 0, 50:60, 60:70]
)
pipeline_log = zero_norm_stack.log.data
assert pipeline_log[0]['method'] == 'Clip'
assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude'
spot_intensities = dartfish.initial_spot_intensities
# assert tht physical coordinates were transferred
assert Coordinates.X in spot_intensities.coords
assert Coordinates.Y in spot_intensities.coords
assert Coordinates.Z in spot_intensities.coords
pipeline_log = spot_intensities.get_log()
assert pipeline_log[0]['method'] == 'Clip'
assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude'
# Test serialization / deserialization of IntensityTable log
with tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as ntf:
tfp = ntf.name
spot_intensities.to_netcdf(tfp)
loaded_intensities = IntensityTable.open_netcdf(tfp)
pipeline_log = loaded_intensities.get_log()
assert pipeline_log[0]['method'] == 'Clip'
assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude'
spots_df = IntensityTable(
spot_intensities.where(spot_intensities[Features.PASSES_THRESHOLDS], drop=True)
).to_features_dataframe()
spots_df['area'] = np.pi * spots_df['radius'] ** 2
# verify number of spots detected
spots_passing_filters = spot_intensities[Features.PASSES_THRESHOLDS].sum()
assert spots_passing_filters == 53
# compare to benchmark data -- note that this particular part of the dataset appears completely
# uncorrelated
cnts_benchmark = pd.read_csv(
'https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv')
min_dist = 0.6
cnts_starfish = spots_df[spots_df.distance <= min_dist].groupby('target').count()['area']
cnts_starfish = cnts_starfish.reset_index(level=0)
cnts_starfish.rename(columns={'target': 'gene', 'area': 'cnt_starfish'}, inplace=True)
# get top 5 genes and verify they are correct
high_expression_genes = cnts_starfish.sort_values('cnt_starfish', ascending=False).head(5)
assert np.array_equal(
high_expression_genes['cnt_starfish'].values,
[7, 3, 2, 2, 2]
)
assert np.array_equal(
high_expression_genes['gene'].values,
['MBP', 'MOBP', 'ADCY8', 'TRIM66', 'SYT6']
)
# verify correlation is accurate for this subset of the image
benchmark_comparison = pd.merge(cnts_benchmark, cnts_starfish, on='gene', how='left')
benchmark_comparison.head(20)
x = benchmark_comparison.dropna().cnt.values
y = benchmark_comparison.dropna().cnt_starfish.values
corrcoef = np.corrcoef(x, y)
corrcoef = corrcoef[0, 1]
assert np.round(corrcoef, 5) == 0.03028
[primary_image, dots_image, nuclei_image, codebook],
[os.fspath(output_path_api)]
)
recipe.run_and_save()
###################################################################################################
# Load up results
# ---------------
#
# We can now load up the results.
import numpy as np
import pandas as pd
from starfish import IntensityTable
from starfish.types import Features
intensity_table = IntensityTable.open_netcdf(os.fspath(output_path_api))
genes, counts = np.unique(
intensity_table.coords[Features.TARGET], return_counts=True)
gene_counts = pd.Series(counts, genes)
print(gene_counts)
###################################################################################################
# Execute the recipe (from the command line)
# ------------------------------------------
#
# We can also execute the recipe from the command line. The command line will look like
cmdline = [
"starfish",
"recipe",
"--recipe",
