def test_wrong_number_of_target_names_raises_error():
"""here we request 3 codes but provide only two names, which should raise an error"""
with pytest.raises(AssertionError):
Codebook.synthetic_one_hot_codebook(n_round=2,
n_channel=2,
n_codes=3,
target_names=list('ab'))
def test_code_length_properly_counts_bit_length_of_codes():
"""
use the codebook factory from another testing module that produces codes with 3 channels and
two rounds. This produces a code which should have length 6.
Then, pass n_round=10 to make the total length 3 * 10 = 30 and verify that the code length is
returned as 30.
"""
codebook_data = codebook_array_factory()
codebook = Codebook.from_code_array(codebook_data)
assert codebook.code_length == 6
codebook_long = Codebook.from_code_array(codebook_data, n_round=10)
assert codebook_long.code_length == 30
def test_from_code_array_throws_exceptions_when_data_does_not_match_channel_or_round_requests(
):
"""
The codebook factory produces codes with 3 channels and 2 rounds. This test provides numbers
larger than that, and the codebook should be expanded to those numbers as a result.
"""
code_array: List = codebook_array_factory()
# should throw an exception, as 3 channels are present in the data
with pytest.raises(ValueError):
Codebook.from_code_array(code_array, n_ch=2, n_round=4)
# should throw an exception, as 2 rounds are present in the data
with pytest.raises(ValueError):
Codebook.from_code_array(code_array, n_ch=3, n_round=1)
def test_from_code_array_has_three_channels_two_rounds_and_two_codes():
"""
Tests that from_code_array loads a small codebook that has the correct size and values
"""
code_array: List = codebook_array_factory()
codebook: Codebook = Codebook.from_code_array(code_array)
assert_sizes(codebook)
def test_unmatched_intensities_and_codebook_table_sizes_throws_value_error():
"""
Codebook and Intensity channel and round number must match. Here we use a codebook with 3
channels, but an IntensityTable with only 2 to verify an error is thrown.
"""
# this codebook has 3 channels
codebook_array = [
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 2,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 0,
Features.CODE_VALUE: 1
}],
Features.TARGET:
'GENE_A'
},
]
codebook = Codebook.from_code_array(codebook_array)
intensities = intensity_table_factory()
with pytest.raises(ValueError):
codebook.metric_decode(intensities,
max_distance=0.5,
min_intensity=1,
norm_order=1)
def parse_codebook(codebook_csv: str) -> Codebook:
"""Parses a codebook csv file provided by SeqFISH developers.
Parameters
----------
codebook_csv : str
The codebook file is expected to contain a matrix whose rows are barcodes and whose columns
are imaging rounds. Column IDs are expected to be sequential, and round identifiers (roman
numerals) are replaced by integer IDs.
Returns
-------
Codebook :
Codebook object in SpaceTx format.
"""
csv: pd.DataFrame = pd.read_csv(codebook_csv, index_col=0)
genes = csv.index.values
data_raw = csv.values
rounds = csv.shape[1]
channels = data_raw.max()
# convert data_raw -> data, where data is genes x channels x rounds
data = np.zeros((len(data_raw), rounds, channels))
for b in range(len(data_raw)):
for i in range(len(data_raw[b])):
if data_raw[b][i] != 0:
data[b][i][data_raw[b][i] - 1] = 1
return Codebook.from_numpy(genes, rounds, channels, data)
def parse_codebook(codebook_csv: str) -> Codebook:
"""Parses a codebook csv file provided by SeqFISH developers.
Parameters
----------
codebook_csv : str
The codebook file is expected to contain a matrix whose rows are barcodes and whose columns
are imaging rounds. Column IDs are expected to be sequential, and round identifiers (roman
numerals) are replaced by integer IDs.
Returns
-------
Codebook :
Codebook object in SpaceTx format.
"""
csv: pd.DataFrame = pd.read_csv(codebook_csv, index_col=0)
integer_round_ids = range(csv.shape[1])
csv.columns = integer_round_ids
mappings = []
for gene, channel_series in csv.iterrows():
mappings.append({
Features.CODEWORD: [{
Axes.ROUND.value: r, Axes.CH.value: c - 1, Features.CODE_VALUE: 1
} for r, c in channel_series.items()],
Features.TARGET: gene
})
return Codebook.from_code_array(mappings)
def codebook_factory() -> Codebook:
"""
Codebook with two codewords describing an experiment with three channels and two imaging rounds.
Both codes have two "on" channels.
"""
codebook_array = [
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 0,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_A"
},
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 2,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_B"
},
]
return Codebook.from_code_array(codebook_array)
def generate_codebook(self, output_dir: str) -> None:
"""Generate and save a codebook from the provided mapping of genes to DNA sequences.
StarMAP codebooks are encoded with the 2-base encoding used for solid sequencing. In this
scheme, multiple pairs of bases map into the same fluorescence channels. This function
exposes this mapping.
Parameters
----------
output_dir : str
directory in which to save the generated codebook. Codebook is saved as "codebook.json"
"""
dinucleotides_to_channels = {
"AT": 4,
"CT": 3,
"GT": 2,
"TT": 1,
"AG": 3,
"CG": 4,
"GG": 1,
"TG": 2,
"AC": 2,
"CC": 1,
"GC": 4,
"TC": 3,
"AA": 1,
"CA": 2,
"GA": 3,
"TA": 4,
}
with open(os.path.join(self.input_dir, "genes.csv"), "r") as f:
codes = [l.strip().split(",") for l in f.readlines()] # List[(gene, dna_barcode), ...]
def iter_dinucleotides(sequence):
i = 0
while i + 1 < len(sequence):
yield sequence[i:i + 2]
i += 1
# construct codebook target mappings
code_array = []
for gene, dna_barcode in codes:
dna_barcode = dna_barcode[::-1] # reverse barcode
spacetx_barcode = [
{
Axes.ROUND.value: r,
Axes.CH.value: dinucleotides_to_channels[dinucleotide],
Features.CODE_VALUE: 1
} for r, dinucleotide in enumerate(iter_dinucleotides(dna_barcode))
]
code_array.append({
Features.CODEWORD: spacetx_barcode,
Features.TARGET: gene
})
codebook = Codebook.from_code_array(code_array)
codebook.to_json(os.path.join(output_dir, "codebook.json"))
def test_codebook_serialization():
"""
Test that codebook can be saved to disk and recovered, and that the recovered codebook is
identical to the one that it was serialized from.
"""
# Create a codebook
codebook_array = codebook_json_data_factory()
codebook = Codebook.from_code_array(codebook_array)
# Dump it to a temporary file
with tempfile.TemporaryDirectory() as directory:
json_codebook = os.path.join(directory, 'codebook.json')
codebook.to_json(json_codebook)
# Retrieve it and test that the data it contains has not changed
codebook_reloaded = Codebook.from_json(json_codebook)
assert codebook_reloaded.equals(codebook)
def two_spot_one_hot_coded_data_factory(
) -> Tuple[Codebook, ImageStack, float]:
"""
Produce a 2-channel 2-round Codebook with two codes and an ImageStack containing one spot from
each code. The spots do not overlap and the data are noiseless.
The encoding of this data is similar to that used in In-situ Sequencing, FISSEQ,
BaristaSeq, STARMAP, MExFISH, or SeqFISH.
Returns
-------
Codebook :
codebook containing codes that match the data
ImageStack :
noiseless ImageStack containing one spot per code in codebook
float :
the maximum intensity found in the created ImageStack
"""
codebook_data = [
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 0,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_A"
},
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 0,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_B"
},
]
codebook = Codebook.from_code_array(codebook_data)
imagestack = create_imagestack_from_codebook(
pixel_dimensions=(10, 100, 100),
spot_coordinates=((4, 10, 90), (5, 90, 10)),
codebook=codebook)
max_intensity = np.max(imagestack.xarray.values)
return codebook, imagestack, max_intensity
def test_target_names_are_incorporated_into_synthetic_codebook():
"""Verify that target names are incorporated in order."""
codebook: Codebook = Codebook.synthetic_one_hot_codebook(
n_round=3,
n_channel=6,
n_codes=2,
target_names=list('ab'),
)
assert np.array_equal(codebook[Features.TARGET], list('ab'))
def two_spot_sparse_coded_data_factory() -> Tuple[Codebook, ImageStack, float]:
"""
Produce a 3-channel 3-round Codebook with two codes and an ImageStack containing one spot from
each code. The spots do not overlap and the data are noiseless.
These spots display sparsity in both rounds and channels, similar to the sparse encoding of
MERFISH
Returns
-------
ImageStack :
noiseless ImageStack containing two spots
"""
codebook_data = [
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 0,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 2,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_A"
},
{
Features.CODEWORD: [{
Axes.ROUND.value: 0,
Axes.CH.value: 1,
Features.CODE_VALUE: 1
}, {
Axes.ROUND.value: 1,
Axes.CH.value: 2,
Features.CODE_VALUE: 1
}],
Features.TARGET:
"GENE_B"
},
]
codebook = Codebook.from_code_array(codebook_data)
imagestack = create_imagestack_from_codebook(
pixel_dimensions=(10, 100, 100),
spot_coordinates=((4, 10, 90), (5, 90, 10)),
codebook=codebook)
max_intensity = np.max(imagestack.xarray.values)
return codebook, imagestack, max_intensity
def test_synthetic_one_hot_codebook_returns_requested_codebook():
"""
Make a request and verify that the size and shape match the request, and
that each round has only one round 'on'.
"""
codebook: Codebook = Codebook.synthetic_one_hot_codebook(n_round=4,
n_channel=2,
n_codes=3)
assert codebook.sizes == {Axes.CH: 2, Axes.ROUND: 4, Features.TARGET: 3}
assert np.all(codebook.sum(Axes.CH.value) ==
1), "the numbers of channels on per round != 1"
def test_from_code_array_expands_codebook_when_provided_n_codes_that_exceeds_array_value(
):
"""
The codebook factory produces codes with 3 channels and 2 rounds. This test provides numbers
larger than that, and the codebook should be expanded to those numbers as a result.
"""
code_array: List = codebook_array_factory()
codebook: Codebook = Codebook.from_code_array(code_array,
n_ch=10,
n_round=4)
assert codebook.sizes[Indices.CH] == 10
assert codebook.sizes[Indices.ROUND] == 4
assert codebook.sizes[Features.TARGET] == 2
def test_create_codebook():
code_array: List = codebook_array_factory()
targets = [x[Features.TARGET] for x in code_array]
# Loop performed by from_code_array
data = np.zeros((2, 3, 2), dtype=np.uint8)
for i, code_dict in enumerate(code_array):
for bit in code_dict[Features.CODEWORD]:
ch = int(bit[Axes.CH])
r = int(bit[Axes.ROUND])
data[i, ch, r] = int(bit[Features.CODE_VALUE])
codebook = Codebook._create_codebook(targets, n_ch=3, n_round=2, data=data)
assert_sizes(codebook)
def test_all_blank_features_yield_non_zero_but_equal_normalized_values(
norm_order, expected_value):
intensity_table = intensity_table_factory()
# zero-out all the values in the IntensityTable
intensity_table.values = np.zeros(4).reshape(1, 2, 2)
normed_intensities, norms = Codebook._normalize_features(
intensity_table, norm_order)
# todo norms here are zero, which seems like the right answer!
assert norms == 0
assert np.all(normed_intensities == expected_value)
def parse_codebook(codebook_csv: str) -> Codebook:
csv: pd.DataFrame = pd.read_csv(codebook_csv, index_col=0)
genes = csv.index.values
data_raw = csv.values
rounds = csv.shape[1]
channels = data_raw.max()
# convert data_raw -> data, where data is genes x channels x rounds
data = np.zeros((len(data_raw), rounds, channels))
for b in range(len(data_raw)):
for i in range(len(data_raw[b])):
if data_raw[b][i] != 0:
data[b][i][data_raw[b][i] - 1] = 1
return Codebook.from_numpy(genes, rounds, channels, data)
def two_spot_informative_blank_coded_data_factory() -> Tuple[Codebook, ImageStack, float]:
"""
Produce a 4-channel 2-round Codebook with two codes and an ImageStack containing one spot from
each code. The spots do not overlap and the data are noiseless.
The encoding of this data is essentially a one-hot encoding, but where one of the channels is a
intentionally and meaningfully "blank".
Returns
-------
Codebook :
codebook containing codes that match the data
ImageStack :
noiseless ImageStack containing one spot per code in codebook
float :
the maximum intensity found in the created ImageStack
"""
codebook_data = [
{
Features.CODEWORD: [
{Axes.ROUND.value: 0, Axes.CH.value: 0, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 1, Axes.CH.value: 1, Features.CODE_VALUE: 1},
# round 3 is blank and channel 3 is not used
],
Features.TARGET: "GENE_A"
},
{
Features.CODEWORD: [
# round 0 is blank and channel 0 is not used
{Axes.ROUND.value: 1, Axes.CH.value: 3, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 2, Axes.CH.value: 2, Features.CODE_VALUE: 1},
],
Features.TARGET: "GENE_B"
},
]
codebook = Codebook.from_code_array(codebook_data)
imagestack = create_imagestack_from_codebook(
pixel_dimensions=(10, 100, 100),
spot_coordinates=((4, 10, 90), (5, 90, 10)),
codebook=codebook
)
max_intensity = np.max(imagestack.xarray.values)
return codebook, imagestack, max_intensity
def test_codebook_loads_from_local_file() -> None:
"""
dumps the codebook data to a temporary json file and reads it back into a Codebook,
verifying that the data has not changed.
"""
# dump codebook to disk
codebook_data: List = codebook_json_data_factory()
with tempfile.TemporaryDirectory() as directory:
codebook_json: str = os.path.join(directory, 'simple_codebook.json')
with open(codebook_json, 'w') as f:
json.dump(codebook_data, f)
# load the codebook
codebook = Codebook.from_json(codebook_json)
assert codebook.sizes[Indices.ROUND] == 2
assert codebook.sizes[Indices.CH] == 3
assert codebook.sizes[Features.TARGET] == 2
def test_normalize_intensities(norm_order, expected_size):
"""
Create a slightly less simple IntensityTable with one "on" tile per feature,
we can again calculate the expected value of the norms. Verify
that the output of the functions correspond to these expectations.
"""
intensity_table = intensity_table_factory()
normed_intensities, norms = Codebook._normalize_features(
intensity_table, norm_order)
assert np.all(norms == expected_size)
# each feature should still have only two non-zero values
assert np.all(
normed_intensities.groupby(Features.AXIS).apply(
lambda x: np.sum(x != 0)) == 2)
# each non-zero value should be equal to 1 / expected_size of the norm.
assert np.all(normed_intensities == intensity_table / norms)
def test_from_code_array_has_three_channels_two_rounds_and_two_codes():
"""
Tests that from_code_array loads a small codebook that has the correct size and values
"""
code_array: List = codebook_array_factory()
codebook: Codebook = Codebook.from_code_array(code_array)
assert codebook.sizes[Indices.CH] == 3
assert codebook.sizes[Indices.ROUND] == 2
assert codebook.sizes[Features.TARGET] == 2
# codebook should have 4 "on" combinations
expected_values = np.zeros((2, 3, 2))
expected_values[0, 0, 0] = 1
expected_values[0, 1, 1] = 1
expected_values[1, 2, 0] = 1
expected_values[1, 1, 1] = 1
assert np.array_equal(codebook.values, expected_values)
def test_normalize_codes(norm_order, expected_size):
"""
Create a simple codebook with two features, each with two "on" sites. For these simple cases,
we can easily calculate the expected value of the norms. (2-norm: sqrt 2, 1-norm: 2), Verify
that the output of the functions correspond to these expectations.
"""
codebook = codebook_factory()
normed_codebook, norms = Codebook._normalize_features(codebook, norm_order)
assert np.all(norms == expected_size)
# each code should still have only two non-zero values
assert np.all(
normed_codebook.groupby(Features.TARGET).apply(
lambda x: np.sum(x != 0)) == 2)
# each non-zero value should be equal to 1 / expected_size of the norm. There are two non-zero
# values and so the sum of the code should be (1 / expected_size) * 2
assert np.all(
normed_codebook.sum((Axes.CH.value,
Axes.ROUND.value)) == (1 / expected_size) * 2)
def convert_codebook(oldbook: Codebook, cycles_conv: Dict[int, int],
channels_conv: List[Dict[int, int]]) -> Codebook:
raw = oldbook.data
targets = np.shape(raw)[0]
rounds = len(cycles_conv)
channels = len(channels_conv[0])
new_data = np.empty((targets, rounds, channels), dtype=int)
for t in range(targets):
for pr in range(len(raw[t])):
# annoying math because dicts are saved for the other direction
pchannel = np.argmax(raw[t][pr])
subChannel = [[
tch for tch, pch in subchannel.items() if pch == pchannel
] for subchannel in channels_conv]
subRound = np.argmax([len(per_round) for per_round in subChannel])
tchannel = subChannel[subRound][0]
tround = [
tr for tr, pround in cycles_conv.items() if pround == pr
][subRound]
# print("channel {}->{} round {}->{}".format(pchannel,tchannel,pr,tround))
new_data[t][tround][tchannel] = 1
return Codebook.from_numpy(oldbook.coords["target"].data, rounds, channels,
new_data)
def test_from_code_array_throws_key_error_with_missing_channel_round_or_value(
):
"""Tests that from_code_array throws errors when it encounters malformed codes"""
code_array: List = codebook_array_factory()
# codebook is now missing a channel
del code_array[0][Features.CODEWORD][0][Indices.ROUND.value]
with pytest.raises(KeyError):
Codebook.from_code_array(code_array)
code_array: List = codebook_array_factory()
del code_array[0][Features.CODEWORD][0][Indices.CH.value]
with pytest.raises(KeyError):
Codebook.from_code_array(code_array)
code_array: List = codebook_array_factory()
del code_array[0][Features.CODEWORD][0][Features.CODE_VALUE]
with pytest.raises(KeyError):
Codebook.from_code_array(code_array)
def format_data(input_dir, output_dir, d):
if not input_dir.endswith("/"):
input_dir += "/"
if not output_dir.endswith("/"):
output_dir += "/"
if d:
url = "http://d1zymp9ayga15t.cloudfront.net/content/Examplezips/ExampleInSituSequencing.zip"
download(input_dir, url)
input_dir += "ExampleInSituSequencing/"
print("Data downloaded to: {}".format(input_dir))
else:
input_dir += "ExampleInSituSequencing/"
print("Using data in : {}".format(input_dir))
def add_codebook(experiment_json_doc):
experiment_json_doc['codebook'] = "codebook.json"
return experiment_json_doc
# the magic numbers here are just for the ISS example data set.
write_experiment_json(
output_dir,
1,
ImageFormat.TIFF,
primary_image_dimensions={
Axes.ROUND: 4,
Axes.CH: 4,
Axes.ZPLANE: 1,
},
aux_name_to_dimensions={
'nuclei': {
Axes.ROUND: 1,
Axes.CH: 1,
Axes.ZPLANE: 1,
},
'dots': {
Axes.ROUND: 1,
Axes.CH: 1,
Axes.ZPLANE: 1,
}
},
primary_tile_fetcher=ISSPrimaryTileFetcher(input_dir),
aux_tile_fetcher={
'nuclei': ISSAuxTileFetcher(os.path.join(input_dir, "DO", "c1.TIF")),
'dots': ISSAuxTileFetcher(os.path.join(input_dir, "DO", "c2.TIF")),
},
postprocess_func=add_codebook,
default_shape=SHAPE
)
codebook_array = [
{
Features.CODEWORD: [
{Axes.ROUND.value: 0, Axes.CH.value: 3, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 1, Axes.CH.value: 3, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 2, Axes.CH.value: 1, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 3, Axes.CH.value: 2, Features.CODE_VALUE: 1}
],
Features.TARGET: "ACTB_human"
},
{
Features.CODEWORD: [
{Axes.ROUND.value: 0, Axes.CH.value: 3, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 1, Axes.CH.value: 1, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 2, Axes.CH.value: 1, Features.CODE_VALUE: 1},
{Axes.ROUND.value: 3, Axes.CH.value: 2, Features.CODE_VALUE: 1}
],
Features.TARGET: "ACTB_mouse"
},
]
codebook = Codebook.from_code_array(codebook_array)
codebook_json_filename = "codebook.json"
codebook.to_json(os.path.join(output_dir, codebook_json_filename))
def test_empty_codebook():
code_array: List = codebook_array_factory()
targets = [x[Features.TARGET] for x in code_array]
codebook = Codebook._empty_codebook(targets, n_channel=3, n_round=2)
assert_sizes(codebook, False)
output_format=output_format,
output_vars=output_vars,
output_dir=output_dir,
file_vars=args.file_vars,
fov_count=args.fov_count,
cache_read_order=args.cache_read_order,
channel_slope=args.channel_slope,
channel_intercept=args.channel_intercept,
fov_offset=args.fov_offset,
round_offset=args.round_offset,
channel_offset=args.channel_offset,
aux_file_formats=args.aux_file_formats,
aux_file_vars=aux_file_vars,
aux_names=args.aux_names,
aux_cache_read_order=aux_cache_read_order,
aux_channel_count=args.aux_channel_count,
aux_channel_slope=args.aux_channel_slope,
aux_channel_intercept=args.aux_channel_intercept,
)
if args.codebook_csv:
codebook = parse_codebook(args.codebook_csv)
elif args.codebook_json:
codebook = Codebook.open_json(args.codebook_json)
else:
print("Can't convert notebook, none provided.")
conv_codebook = convert_codebook(codebook, cycles_conv, channels_conv)
codebook.to_json(output_dir + "pround_codebook.json")
conv_codebook.to_json(output_dir + "codebook.json")
def test_from_code_array_throws_exception_when_data_is_improperly_formatted():
code_array: List = codebook_array_factory()
code_array[0][Features.CODEWORD][0] = ('I should be a dict, oops!', )
with pytest.raises(TypeError):
Codebook.from_code_array(code_array, n_ch=3, n_round=1)
def codebook(self) -> Codebook:
return Codebook.synthetic_one_hot_codebook(self.n_round, self.n_ch,
self.n_codes)
