def _cli(cls, args, print_help=False):
"""Runs the pixel finder component based on parsed arguments."""
if args.pixel_finder_algorithm_class is None or print_help:
cls.pixel_finder_group.print_help()
cls.pixel_finder_group.exit(status=2)
print('Detecting Pixels...')
s = Stack()
s.read(args.input)
instance = args.pixel_finder_algorithm_class(**vars(args))
# TODO does this work?
pixel_attributes, encoded_pixels = instance.find(s)
# TODO ambrosejcarr: this still needs to be added back.
# if args.show:
# encoded_pixels.show(figsize=(10, 10))
path = os.path.join(args.output, 'pixels.geojson')
print(f"Writing | pixels geojson to: {path}")
pixel_attributes.save_geojson(path)
path = os.path.join(args.output, 'pixels.json')
print(f"Writing | spot_id | x | y | z | to: {path}")
pixel_attributes.save(path)
path = os.path.join(args.output, 'encoder_table.json')
print(f"Writing | spot_id | hyb | ch | val | to: {path}")
encoded_pixels.save(path)
def merfish_stack() -> Stack:
"""retrieve MERFISH testing data from cloudfront and expose it at the module level
Notes
-----
Because download takes time, this fixture runs once per session -- that is, the download is run only once.
Returns
-------
Stack :
starfish.io.Stack object containing MERFISH data
"""
s = Stack()
s.read('https://s3.amazonaws.com/czi.starfish.data.public/20180607/test/MERFISH/fov_001/experiment_new.json')
return deepcopy(s)
def _cli(cls, args, print_help=False):
"""Runs the filter component based on parsed arguments."""
if args.filter_algorithm_class is None or print_help:
cls.filter_group.print_help()
cls.filter_group.exit(status=2)
print('Filtering images ...')
s = Stack()
s.read(args.input)
instance = args.filter_algorithm_class(**vars(args))
instance.filter(s)
s.write(args.output)
def _cli(cls, args, print_help=False):
"""Runs the registration component based on parsed arguments."""
if args.registration_algorithm_class is None or print_help:
cls.register_group.print_help()
cls.register_group.exit(status=2)
instance = args.registration_algorithm_class(**vars(args))
from starfish.io import Stack
print('Registering ...')
s = Stack()
s.read(args.input)
instance.register(s)
s.write(args.output)
from showit import image, tile # EPY: END code # EPY: START markdown # ## Raw Data # # The raw data can be downloaded and formatted for analysis by running: ```python examples/get_iss_data.py ><raw data directory> <output directory> --d 1``` from the Starfish directory # EPY: END markdown # EPY: START code from starfish.io import Stack # replace <output directory> with where you saved the formatted data to with the above script in_json = '<output directory>/org.json' s = Stack() s.read(in_json) tile(s.squeeze(), size=10) # EPY: END code # EPY: START code image(s.aux_dict['dots'], size=10) # EPY: END code # EPY: START markdown # ## Register # EPY: END markdown # EPY: START code from starfish.registration._fourier_shift import compute_shift, shift_im
#
# ## Load tiff stack and visualize one field of view
# EPY: END markdown
# EPY: START code
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from showit import image, tile
import pprint
# EPY: ESCAPE %matplotlib inline
from starfish.io import Stack
from starfish.constants import Indices
s = Stack()
s.read(
'https://dmf0bdeheu4zf.cloudfront.net/20180611/ISS/fov_001/experiment.json'
)
# s.image.squeeze() simply converts the 4D tensor H*C*X*Y into a list of len(H*C) image planes for rendering by 'tile'
tile(s.image.squeeze())
# EPY: END code
# EPY: START markdown
# ## Show input file format that specifies how the tiff stack is organized
#
# The stack contains multiple single plane images, one for each color channel, 'ch', (columns in above image) and hybridization round, 'hyb', (rows in above image). This protocol assumes that genes are encoded with a length 4 quatenary barcode that can be read out from the images. Each hybridization encodes a position in the codeword. The maximum signal in each color channel (columns in the above image) corresponds to a letter in the codeword. The channels, in order, correspond to the letters: 'T', 'G', 'C', 'A'. The goal is now to process these image data into spatially organized barcodes, e.g., ACTG, which can then be mapped back to a codebook that specifies what gene this codeword corresponds to.
# EPY: END markdown
# EPY: START code
pp = pprint.PrettyPrinter(indent=2)
# The numpy array can be accessed through Stack.image.numpy\_array (public method, read only) or Stack.image.\_data (read and write)
# EPY: END markdown
# EPY: START code
codebook = pd.read_json(
'https://dmf0bdeheu4zf.cloudfront.net/20180606/allen_smFISH/fov_001/codebook.json'
)
codebook
# EPY: END code
# EPY: START markdown
# We're ready now to load the experiment into starfish (This experiment is big, it takes a few minutes):
# EPY: END markdown
# EPY: START code
s = Stack()
s.read(experiment_json)
# EPY: END code
# EPY: START markdown
# All of our implemented operations leverage the `Stack.image.apply` method to apply a single function over each of the tiles or volumes in the FOV, depending on whether the method accepts a 2d or 3d array. Below, we're clipping each image independently at the 10th percentile. I've placed the imports next to the methods so that you can easily locate the code, should you want to look under the hood and understand what parameters have been chosen.
#
# The verbose flag for our apply loops could use a bit more refinement. We should be able to tell it how many images it needs to process from looking at the image stack, but for now it's dumb so just reports the number of tiles or volumes it's processed. This FOV has 102 images over 3 volumes.
# EPY: END markdown
# EPY: START code
from starfish.pipeline.filter import Filter
s_clip = Filter.Clip(p_min=10, p_max=100, verbose=True)
s_clip.filter(s.image)
# EPY: END code
def synthesize() -> Tuple[Stack, list]:
"""Synthesize synthetic spatial image-based transcriptomics data
Returns
-------
Stack :
starfish Stack containing synthetic spots
list :
codebook matching the synthetic data
"""
# set random seed so that data is consistent across tests
random.seed(2)
np.random.seed(2)
NUM_HYB = 4
NUM_CH = 2
NUM_Z = 1
HEIGHT = 100
WIDTH = 100
assert WIDTH == HEIGHT # for compatibility with the parameterization of the code
def choose(n, k):
if n == k:
return [[1] * k]
subsets = [[0] + a for a in choose(n - 1, k)]
if k > 0:
subsets += [[1] + a for a in choose(n - 1, k - 1)]
return subsets
def graham_sloane_codes(n):
# n is length of codeword
# number of on bits is 4
def code_sum(codeword):
return sum([i * c for i, c in enumerate(codeword)]) % n
return [c for c in choose(n, 4) if code_sum(c) == 0]
p = {
# number of on bits (not used with current codebook)
'N_high': 4,
# length of barcode
'N_barcode': NUM_CH * NUM_HYB,
# mean number of flourophores per transcripts - depends on amplification strategy (e.g HCR, bDNA)
'N_flour': 200,
# mean number of photons per flourophore - depends on exposure time, bleaching rate of dye
'N_photons_per_flour': 50,
# mean number of background photons per pixel - depends on tissue clearing and autoflourescence
'N_photon_background': 1000,
# quantum efficiency of the camera detector units number of electrons per photon
'detection_efficiency': .25,
# camera read noise per pixel in units electrons
'N_background_electrons': 1,
# number of RNA puncta; keep this low to reduce overlap probability
'N_spots': 20,
# height and width of image in pixel units
'N_size': WIDTH,
# standard devitation of gaussian in pixel units
'psf': 2,
# dynamic range of camera sensor 37,000 assuming a 16-bit AD converter
'graylevel': 37000.0 / 2 ** 16,
# 16-bit AD converter
'bits': 16
}
codebook = graham_sloane_codes(p['N_barcode'])
def generate_spot(p):
position = rand(2)
gene = random.choice(range(len(codebook)))
barcode = array(codebook[gene])
photons = [poisson(p['N_photons_per_flour']) * poisson(p['N_flour']) * b for b in barcode]
return DataFrame({'position': [position], 'barcode': [barcode], 'photons': [photons], 'gene': gene})
# right now there is no jitter on x-y positions of the spots, we might want to make it a vector
spots = concat([generate_spot(p) for _ in range(p['N_spots'])]) # type: ignore
image = zeros((p['N_barcode'], p['N_size'], p['N_size'],))
for s in spots.itertuples():
image[:, int(p['N_size'] * s.position[0]), int(p['N_size'] * s.position[1])] = s.photons
image_with_background = image + poisson(p['N_photon_background'], size=image.shape)
filtered = array([gaussian(im, p['psf']) for im in image_with_background])
filtered = filtered * p['detection_efficiency'] + normal(scale=p['N_background_electrons'], size=filtered.shape)
signal = np.array([(x / p['graylevel']).astype(int).clip(0, 2 ** p['bits']) for x in filtered])
def select_uint_dtype(array):
"""choose appropriate dtype based on values of an array"""
max_val = np.max(array)
for dtype in [np.uint8, np.uint16, np.uint32, np.uint64]:
if max_val <= dtype(-1):
return array.astype(dtype)
raise ValueError('value exceeds dynamic range of largest numpy type')
corrected_signal = select_uint_dtype(signal)
rescaled_signal: np.ndarray = rescale_intensity(corrected_signal)
# set up the tile set
image_data = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: NUM_HYB,
Indices.CH: NUM_CH,
Indices.Z: NUM_Z,
},
default_tile_shape=(HEIGHT, WIDTH),
)
# fill the TileSet
experiment_indices = list(product(range(NUM_HYB), range(NUM_CH), range(NUM_Z)))
for i, (hyb, ch, z) in enumerate(experiment_indices):
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
}
)
tile.numpy_array = rescaled_signal[i]
image_data.add_tile(tile)
data_stack = ImageStack(image_data)
# make a max projection and pretend that's the dots image, which we'll create another ImageStack for this
dots_data = TileSet(
{Coordinates.X, Coordinates.Y, Indices.HYB, Indices.CH, Indices.Z},
{
Indices.HYB: 1,
Indices.CH: 1,
Indices.Z: 1,
},
default_tile_shape=(HEIGHT, WIDTH),
)
tile = Tile(
{
Coordinates.X: (0.0, 0.001),
Coordinates.Y: (0.0, 0.001),
Coordinates.Z: (0.0, 0.001),
},
{
Indices.HYB: 0,
Indices.CH: 0,
Indices.Z: 0,
}
)
tile.numpy_array = np.max(rescaled_signal, 0)
dots_data.add_tile(tile)
dots_stack = ImageStack(dots_data)
# TODO can we mock up a nuclei image somehow?
# put the data together into a top-level Stack
results = Stack.from_data(data_stack, aux_dict={'dots': dots_stack})
# make the codebook(s)
codebook = []
for _, code_record in spots.iterrows():
codeword = []
for code_value, (hyb, ch, z) in zip(code_record['barcode'], experiment_indices):
if code_value != 0:
codeword.append({
Indices.HYB: hyb,
Indices.CH: ch,
Indices.Z: z,
"v": code_value
})
codebook.append(
{
'codeword': codeword,
'gene_name': code_record['gene']
}
)
return results, codebook
from showit import image, tile # EPY: END code # EPY: START markdown # ## Raw Data # # The raw data can be downloaded and formatted for analysis by running: ```python examples/get_iss_data.py ><raw data directory> <output directory> --d 1``` from the Starfish directory # EPY: END markdown # EPY: START code from starfish.io import Stack # replace <output directory> with where you saved the formatted data to with the above script in_json = '<output directory>/org.json' s = Stack() s.read(in_json) tile(s.image.squeeze(), size=10); # EPY: END code # EPY: START code image(s.auxiliary_images['dots'], size=10) # EPY: END code # EPY: START markdown # ## Register # EPY: END markdown # EPY: START code from starfish.pipeline.registration import Registration
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import scoreatpercentile
from showit import image, tile
from starfish.constants import Indices
from starfish.io import Stack
from starfish.viz import tile_lims
# EPY: ESCAPE %matplotlib inline
# EPY: END code
# EPY: START code
# load the data from cloudfront
s = Stack()
s.read('https://dmf0bdeheu4zf.cloudfront.net/MERFISH/fov_001/experiment.json')
# EPY: END code
# EPY: START code
# data from one FOV correspond to 16 single plane images as shown here (see below for details)
tile(s.image.squeeze())
# EPY: END code
# EPY: START markdown
# Individual hybridization rounds and channels can also be visualized
# EPY: END markdown
# EPY: START code
# show all hybridization rounds of channel 0
s.image.show_stack({Indices.CH: 0})