def test_no_tiling(self):
a = make_random_data()
for long in False, True:
all_sigs = list(calc_features(a, self.name, long=long))
self.assertEqual(len(all_sigs), 1)
sigs = all_sigs[0]
rec = to_avro(sigs)
for k in 'img_path', 'series', 'z', 'c', 't':
rec[k] = getattr(self, k)
try:
pyavroc_emu.AvroSerializer(Signatures).serialize(rec)
except AvroException as e:
self.fail("Could not serialize record: %s" % e)
self.assertEquals(rec["version"], sigs.feature_set_version)
self.assertEquals(rec["name"], self.name)
self.assertEquals((rec["x"], rec["y"]), (0, 0))
self.assertEquals((rec["h"], rec["w"]), a.shape)
fmap = dict(izip(sigs.feature_names, sigs.values))
for fname, (vname, idx) in FEATURE_NAMES.iteritems():
v = fmap.get(fname)
if v is None:
assert not long
self.assertEqual(len(rec[vname]), 0)
else:
self.assertEqual(rec[vname][idx], v)
def setUp(self):
name = "img_0"
img_path = "/bar/spam/img_0.tif"
series = z = c = t = 0
array_slice = {
"dtype": "UINT8",
"little_endian": True,
"shape": [4, 4, 1, 2, 3],
"offsets": [0, 0, z, c, t],
"deltas": [4, 4, 1, 1, 1],
"data": ''.join(chr(_) for _ in xrange(16))
}
signatures = {
"version": "3.2",
"name": name,
"img_path": img_path,
"series": series, "z": z, "c": c, "t": t,
"x": 0, "y": 0, "w": 400, "h": 300,
}
for vname, idx in FEATURE_NAMES.itervalues():
signatures.setdefault(vname, []).append(float(idx))
self.record_map = {
"BioImgPlane": {
"name": name,
"img_path": img_path,
"dimension_order": "XYZCT",
"series": series,
"pixel_data": array_slice,
},
"Signatures": signatures
}
def test_tiling(self):
a = make_random_data()
w, h = 3, 4
s = list(calc_features(a, self.name, w=w, h=h))
self.assertEqual(len(s), 6)
r = [to_avro(_) for _ in s]
for i in xrange(6):
for k in 'img_path', 'series', 'z', 'c', 't':
r[i][k] = getattr(self, k)
try:
[pyavroc_emu.AvroSerializer(Signatures).serialize(_) for _ in r]
except AvroException as e:
self.fail("Could not serialize record: %s" % e)
for i in xrange(6):
self.assertEquals(r[i]["version"], s[i].feature_set_version)
self.assertEquals(r[i]["name"], self.name)
fmap = dict(izip(s[i].feature_names, s[i].values))
for fname, (vname, idx) in FEATURE_NAMES.iteritems():
v = fmap.get(fname)
if v is None:
self.assertEqual(len(r[i][vname]), 0)
else:
self.assertEqual(r[i][vname][idx], v)
self.assertEqual(
(r[0]["x"], r[0]["y"], r[0]["w"], r[0]["h"]), (0, 0, 3, 4))
self.assertEqual(
(r[1]["x"], r[1]["y"], r[1]["w"], r[1]["h"]), (3, 0, 3, 4))
self.assertEqual(
(r[2]["x"], r[2]["y"], r[2]["w"], r[2]["h"]), (6, 0, 2, 4))
self.assertEqual(
(r[3]["x"], r[3]["y"], r[3]["w"], r[3]["h"]), (0, 4, 3, 2))
self.assertEqual(
(r[4]["x"], r[4]["y"], r[4]["w"], r[4]["h"]), (3, 4, 3, 2))
self.assertEqual(
(r[5]["x"], r[5]["y"], r[5]["w"], r[5]["h"]), (6, 4, 2, 2))
def setUp(self):
name = "img_0"
img_path = "/bar/spam/img_0.tif"
series = z = c = t = 0
array_slice = {
"dtype": "UINT8",
"little_endian": True,
"shape": [4, 4, 1, 2, 3],
"offsets": [0, 0, z, c, t],
"deltas": [4, 4, 1, 1, 1],
"data": ''.join(chr(_) for _ in xrange(16))
}
signatures = {
"version": "3.2",
"name": name,
"img_path": img_path,
"series": series, "z": z, "c": c, "t": t,
"x": 0, "y": 0, "w": 400, "h": 300,
}
for vname, idx in FEATURE_NAMES.itervalues():
signatures.setdefault(vname, []).append(float(idx))
self.record_map = {
"BioImgPlane": {
"name": name,
"img_path": img_path,
"dimension_order": "XYZCT",
"series": series,
"pixel_data": array_slice,
},
"Signatures": signatures
}
def get_data(fn, axis, feature=None, x=None, y=None):
other_axes = [_ for _ in AXES if _ != axis]
data = {}
for r in iter_records(fn):
if (x is not None and r['x'] != x) or (y is not None and r['y'] != y):
continue
k1 = tuple(r[_] for _ in other_axes)
k2 = (r['x'], r['y'])
for name, idx in FEATURE_NAMES.itervalues():
if feature and name != feature:
continue
try:
subv = r[name]
except KeyError:
continue # "short" feature set, this subvector is missing
k3 = (name, idx)
data.setdefault(
k1, {}).setdefault(
k2, {}).setdefault(
k3, []).append((r[axis], subv[idx]))
for k1, v1 in data.iteritems():
for k2, v2 in v1.iteritems():
for k3, v3 in v2.iteritems():
v2[k3] = [_[1] for _ in sorted(v3)]
return data
def test_tiling(self):
a = make_random_data()
w, h = 3, 4
s = list(calc_features(a, self.name, w=w, h=h))
self.assertEqual(len(s), 6)
r = [to_avro(_) for _ in s]
for i in xrange(6):
for k in 'img_path', 'series', 'z', 'c', 't':
r[i][k] = getattr(self, k)
try:
[pyavroc_emu.AvroSerializer(Signatures).serialize(_) for _ in r]
except AvroException as e:
self.fail("Could not serialize record: %s" % e)
for i in xrange(6):
self.assertEquals(r[i]["version"], s[i].feature_set_version)
self.assertEquals(r[i]["name"], self.name)
fmap = dict(izip(s[i].feature_names, s[i].values))
for fname, (vname, idx) in FEATURE_NAMES.iteritems():
v = fmap.get(fname)
if v is None:
self.assertEqual(len(r[i][vname]), 0)
else:
self.assertEqual(r[i][vname][idx], v)
self.assertEqual(
(r[0]["x"], r[0]["y"], r[0]["w"], r[0]["h"]), (0, 0, 3, 4))
self.assertEqual(
(r[1]["x"], r[1]["y"], r[1]["w"], r[1]["h"]), (3, 0, 3, 4))
self.assertEqual(
(r[2]["x"], r[2]["y"], r[2]["w"], r[2]["h"]), (6, 0, 2, 4))
self.assertEqual(
(r[3]["x"], r[3]["y"], r[3]["w"], r[3]["h"]), (0, 4, 3, 2))
self.assertEqual(
(r[4]["x"], r[4]["y"], r[4]["w"], r[4]["h"]), (3, 4, 3, 2))
self.assertEqual(
(r[5]["x"], r[5]["y"], r[5]["w"], r[5]["h"]), (6, 4, 2, 2))
def test_no_tiling(self):
a = make_random_data()
for long in False, True:
all_sigs = list(calc_features(a, self.name, long=long))
self.assertEqual(len(all_sigs), 1)
sigs = all_sigs[0]
rec = to_avro(sigs)
for k in 'img_path', 'series', 'z', 'c', 't':
rec[k] = getattr(self, k)
try:
pyavroc_emu.AvroSerializer(Signatures).serialize(rec)
except AvroException as e:
self.fail("Could not serialize record: %s" % e)
self.assertEquals(rec["version"], sigs.feature_set_version)
self.assertEquals(rec["name"], self.name)
self.assertEquals((rec["x"], rec["y"]), (0, 0))
self.assertEquals((rec["h"], rec["w"]), a.shape)
fmap = dict(izip(sigs.feature_names, sigs.values))
for fname, (vname, idx) in FEATURE_NAMES.iteritems():
v = fmap.get(fname)
if v is None:
assert not long
self.assertEqual(len(rec[vname]), 0)
else:
self.assertEqual(rec[vname][idx], v)
def get_data(fn, axis, feature=None, x=None, y=None):
other_axes = [_ for _ in AXES if _ != axis]
data = {}
for r in iter_records(fn):
if (x is not None and r['x'] != x) or (y is not None and r['y'] != y):
continue
k1 = tuple(r[_] for _ in other_axes)
k2 = (r['x'], r['y'])
for name, idx in FEATURE_NAMES.itervalues():
if feature and name != feature:
continue
try:
subv = r[name]
except KeyError:
continue # "short" feature set, this subvector is missing
k3 = (name, idx)
data.setdefault(
k1, {}).setdefault(
k2, {}).setdefault(
k3, []).append((r[axis], subv[idx]))
for k1, v1 in data.iteritems():
for k2, v2 in v1.iteritems():
for k3, v3 in v2.iteritems():
v2[k3] = [_[1] for _ in sorted(v3)]
return data
def to_avro(signatures):
rec = dict((_[0], []) for _ in FEATURE_NAMES.itervalues())
for fname, value in izip(signatures.feature_names, signatures.values):
vname, idx = FEATURE_NAMES[fname]
rec[vname].append((idx, value))
for vname, tuples in rec.iteritems():
rec[vname] = [_[1] for _ in sorted(tuples)]
rec["version"] = signatures.feature_set_version
rec["name"] = signatures.basename
for k in "x", "y", "w", "h":
rec[k] = getattr(signatures, k)
return rec
def to_avro(signatures):
rec = dict((_[0], []) for _ in FEATURE_NAMES.itervalues())
for fname, value in izip(signatures.feature_names, signatures.values):
vname, idx = FEATURE_NAMES[fname]
rec[vname].append((idx, value))
for vname, tuples in rec.iteritems():
rec[vname] = [_[1] for _ in sorted(tuples)]
rec["version"] = signatures.feature_set_version
rec["name"] = signatures.basename
for k in "x", "y", "w", "h":
rec[k] = getattr(signatures, k)
return rec
import matplotlib.pyplot as plt
try:
from pyavroc import AvroFileReader
except ImportError:
from pyfeatures.pyavroc_emu import AvroFileReader
warnings.warn("pyavroc not found, using standard avro lib\n")
from pyfeatures.feature_names import FEATURE_NAMES
AXES = "z", "c", "t"
AX_DESC = {
"z": "Z-sections",
"c": "Channels",
"t": "Timepoints",
}
FV_NAMES = frozenset(_[0] for _ in FEATURE_NAMES.itervalues())
def iter_records(fn):
ext = os.path.splitext(fn)[-1]
if ext == ".db":
with closing(shelve.open(fn, flag="r")) as shelf:
for r in shelf.itervalues():
yield r
elif ext == ".pickle":
with open(fn) as f:
for r in cPickle.load(f):
yield r
else:
with open(fn) as f:
reader = AvroFileReader(f)
try:
from pyavroc import AvroFileReader
except ImportError:
from pyfeatures.pyavroc_emu import AvroFileReader
warnings.warn("pyavroc not found, using standard avro lib\n")
from pyfeatures.feature_names import FEATURE_NAMES
AXES = "z", "c", "t"
AX_DESC = {
"z": "Z-sections",
"c": "Channels",
"t": "Timepoints",
}
FV_NAMES = frozenset(_[0] for _ in FEATURE_NAMES.itervalues())
def iter_records(fn):
ext = os.path.splitext(fn)[-1]
if ext == ".db":
with closing(shelve.open(fn, flag="r")) as shelf:
for r in shelf.itervalues():
yield r
elif ext == ".pickle":
with open(fn) as f:
for r in cPickle.load(f):
yield r
else:
with open(fn) as f:
reader = AvroFileReader(f)