def verify_results(self, tempdir: Path): intensities = IntensityTable.open_netcdf( os.fspath(tempdir / "decoded_spots.nc")) genes, counts = np.unique(intensities.coords[Features.TARGET], return_counts=True) gene_counts = pd.Series(counts, genes) assert gene_counts['ACTB'] == 9 assert gene_counts['GAPDH'] == 9
def _load_data(fov_data, exp): """ Load a field of view dataset Parameters ---------- fov_data : pd.DataFrame Table of file locations for the spot results and mask label image corresponding to the dataset to be viewed exp : Experiment Experiment file corresponding to the data analysis Returns ------- im_max_proj : np.ndarray Image used for the spot detection points : List[Dict] List of dictionaries containing the points and target names for the points to be rendered mask_im : np.ndarray Label image for the segmentation mask """ # Get the RNAscope images im = exp[fov_data.fov_name].get_image("primary") im_max_proj = np.squeeze(im.max_proj(Axes.ZPLANE).xarray.values) # Get the spots spots_file = fov_data.spot_file it = IntensityTable.open_netcdf(spots_file) points = _get_points(it, exp) # Get the segmentation mask mask_file = fov_data.mask_file mask_im = io.imread(mask_file) return im_max_proj, points, mask_im
def test_dartfish_pipeline_cropped_data(tmpdir): # set random seed to errors provoked by optimization functions np.random.seed(777) dartfish = __import__('DARTFISH') primary_image = dartfish.imgs expected_primary_image = np.array( [[1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 4.57770657e-05, 0.00000000e+00, 0.00000000e+00, 3.05180438e-05], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.05180438e-05, 0.00000000e+00], [0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 3.05180438e-05, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00], [0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 3.05180438e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00], [0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00], [1.52590219e-05, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00], [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.52590219e-05, 1.52590219e-05, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00]], dtype=np.float32 ) assert primary_image.xarray.dtype == np.float32 assert np.allclose( primary_image.xarray[0, 0, 0, 50:60, 60:70], expected_primary_image ) normalized_image = dartfish.norm_imgs expected_normalized_image = np.array( [[0.01960784, 0., 0., 0., 0., 0., 0.05882353, 0., 0., 0.03921569], [0., 0., 0., 0., 0.01960784, 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.01960784, 0., 0., 0., 0.03921569, 0.], [0., 0., 0.01960784, 0., 0., 0.01960784, 0., 0.01960784, 0., 0.], [0., 0., 0., 0., 0., 0., 0.03921569, 0., 0.01960784, 0.], [0., 0., 0.01960784, 0., 0.03921569, 0., 0., 0., 0., 0.01960784], [0., 0., 0., 0., 0., 0.01960784, 0., 0., 0., 0.], [0., 0.01960784, 0., 0., 0.01960784, 0., 0., 0., 0., 0.], [0.01960784, 0.01960784, 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0.01960784, 0.01960784, 0., 0., 0.]], dtype=np.float32, ) assert normalized_image.xarray.dtype == np.float32 assert np.allclose( normalized_image.xarray[0, 0, 0, 50:60, 60:70], expected_normalized_image ) zero_norm_stack = dartfish.filtered_imgs expected_zero_normalized_image = np.array( [[0.01960784, 0., 0., 0., 0., 0., 0.05882353, 0., 0., 0.03921569], [0., 0., 0., 0., 0.01960784, 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.01960784, 0., 0., 0., 0., 0.], [0., 0., 0.01960784, 0., 0., 0., 0., 0.01960784, 0., 0.], [0., 0., 0., 0., 0., 0., 0.03921569, 0., 0.01960784, 0.], [0., 0., 0.01960784, 0., 0., 0., 0., 0., 0., 0.01960784], [0., 0., 0., 0., 0., 0.01960784, 0., 0., 0., 0.], [0., 0.01960784, 0., 0., 0., 0., 0., 0., 0., 0.], [0.01960784, 0., 0., 0., 0., 0., 0., 0., 0., 0.], [0., 0., 0., 0., 0., 0., 0.01960784, 0., 0., 0.]], dtype=np.float32 ) assert np.allclose( expected_zero_normalized_image, zero_norm_stack.xarray[0, 0, 0, 50:60, 60:70] ) pipeline_log = zero_norm_stack.log.data assert pipeline_log[0]['method'] == 'Clip' assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude' spot_intensities = dartfish.initial_spot_intensities # assert tht physical coordinates were transferred assert Coordinates.X in spot_intensities.coords assert Coordinates.Y in spot_intensities.coords assert Coordinates.Z in spot_intensities.coords pipeline_log = spot_intensities.get_log() assert pipeline_log[0]['method'] == 'Clip' assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude' # Test serialization / deserialization of IntensityTable log with tempfile.NamedTemporaryFile(dir=tmpdir, delete=False) as ntf: tfp = ntf.name spot_intensities.to_netcdf(tfp) loaded_intensities = IntensityTable.open_netcdf(tfp) pipeline_log = loaded_intensities.get_log() assert pipeline_log[0]['method'] == 'Clip' assert pipeline_log[1]['method'] == 'ZeroByChannelMagnitude' spots_df = IntensityTable( spot_intensities.where(spot_intensities[Features.PASSES_THRESHOLDS], drop=True) ).to_features_dataframe() spots_df['area'] = np.pi * spots_df['radius'] ** 2 # verify number of spots detected spots_passing_filters = spot_intensities[Features.PASSES_THRESHOLDS].sum() assert spots_passing_filters == 53 # compare to benchmark data -- note that this particular part of the dataset appears completely # uncorrelated cnts_benchmark = pd.read_csv( 'https://d2nhj9g34unfro.cloudfront.net/20181005/DARTFISH/fov_001/counts.csv') min_dist = 0.6 cnts_starfish = spots_df[spots_df.distance <= min_dist].groupby('target').count()['area'] cnts_starfish = cnts_starfish.reset_index(level=0) cnts_starfish.rename(columns={'target': 'gene', 'area': 'cnt_starfish'}, inplace=True) # get top 5 genes and verify they are correct high_expression_genes = cnts_starfish.sort_values('cnt_starfish', ascending=False).head(5) assert np.array_equal( high_expression_genes['cnt_starfish'].values, [7, 3, 2, 2, 2] ) assert np.array_equal( high_expression_genes['gene'].values, ['MBP', 'MOBP', 'ADCY8', 'TRIM66', 'SYT6'] ) # verify correlation is accurate for this subset of the image benchmark_comparison = pd.merge(cnts_benchmark, cnts_starfish, on='gene', how='left') benchmark_comparison.head(20) x = benchmark_comparison.dropna().cnt.values y = benchmark_comparison.dropna().cnt_starfish.values corrcoef = np.corrcoef(x, y) corrcoef = corrcoef[0, 1] assert np.round(corrcoef, 5) == 0.03028
[primary_image, dots_image, nuclei_image, codebook], [os.fspath(output_path_api)] ) recipe.run_and_save() ################################################################################################### # Load up results # --------------- # # We can now load up the results. import numpy as np import pandas as pd from starfish import IntensityTable from starfish.types import Features intensity_table = IntensityTable.open_netcdf(os.fspath(output_path_api)) genes, counts = np.unique( intensity_table.coords[Features.TARGET], return_counts=True) gene_counts = pd.Series(counts, genes) print(gene_counts) ################################################################################################### # Execute the recipe (from the command line) # ------------------------------------------ # # We can also execute the recipe from the command line. The command line will look like cmdline = [ "starfish", "recipe", "--recipe",