def make_fid(name, mode, userblock_size, fapl, fcpl=None):
""" Get a new FileID by opening or creating a file.
Also validates mode argument."""
if userblock_size is not None:
if mode in ("r", "r+"):
raise ValueError("User block may only be specified " "when creating a file")
try:
userblock_size = int(userblock_size)
except (TypeError, ValueError):
raise ValueError("User block size must be an integer")
if fcpl is None:
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_userblock(userblock_size)
if mode == "r":
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
elif mode == "r+":
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
elif mode == "w-":
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode == "w":
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
elif mode == "a":
# Open in append mode (read/write).
# If that fails, create a new file only if it won't clobber an
# existing one (ACC_EXCL)
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode is None:
# Try to open in append mode (read/write).
# If that fails, try readonly, and finally create a new file only
# if it won't clobber an existing file (ACC_EXCL).
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
except IOError:
try:
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
else:
raise ValueError("Invalid mode; must be one of r, r+, w, w-, a")
try:
if userblock_size is not None:
existing_fcpl = fid.get_create_plist()
if existing_fcpl.get_userblock() != userblock_size:
raise ValueError(
"Requested userblock size (%d) does not match that of existing file (%d)"
% (userblock_size, existing_fcpl.get_userblock())
)
except:
fid.close()
raise
return fid
def make_fid(name, mode, userblock_size, fapl, fcpl=None):
""" Get a new FileID by opening or creating a file.
Also validates mode argument."""
if userblock_size is not None:
if mode in ('r', 'r+'):
raise ValueError("User block may only be specified "
"when creating a file")
try:
userblock_size = int(userblock_size)
except (TypeError, ValueError):
raise ValueError("User block size must be an integer")
if fcpl is None:
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_userblock(userblock_size)
if mode == 'r':
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
elif mode == 'r+':
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
elif mode == 'w-':
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode == 'w':
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
elif mode == 'a':
# Open in append mode (read/write).
# If that fails, create a new file only if it won't clobber an
# existing one (ACC_EXCL)
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode is None:
# Try to open in append mode (read/write).
# If that fails, try readonly, and finally create a new file only
# if it won't clobber an existing file (ACC_EXCL).
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
except IOError:
try:
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
else:
raise ValueError("Invalid mode; must be one of r, r+, w, w-, a")
try:
if userblock_size is not None:
existing_fcpl = fid.get_create_plist()
if existing_fcpl.get_userblock() != userblock_size:
raise ValueError(
"Requested userblock size (%d) does not match that of existing file (%d)"
% (userblock_size, existing_fcpl.get_userblock()))
except:
fid.close()
raise
return fid
def ensureHdf(file, mode):
if isinstance(file, str) or file is None:
driver, driverKwds = None, {}
if file is None:
file = tempfile.mktemp(suffix=".hdf")
driver, driverKwds = "core", {"backing_store": False}
dirname = os.path.dirname(os.path.abspath(file))
if not os.path.exists(dirname):
os.makedirs(dirname)
return h5py.File(file,
mode,
libver="earliest",
driver=driver,
**driverKwds)
elif isinstance(file, bytes):
fapl = h5p.create(h5p.FILE_ACCESS)
fapl.set_fapl_core()
fapl.set_file_image(file)
fid = h5f.open(tempfile.mktemp(suffix=".hdf").encode(),
h5f.ACC_RDONLY,
fapl=fapl)
return h5py.File(fid)
else:
return file
def make_fid(name, mode, plist):
""" Get a new FileID by opening or creating a file.
Also validates mode argument."""
if mode == 'r':
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=plist)
elif mode == 'r+':
fid = h5f.open(name, h5f.ACC_RDWR, fapl=plist)
elif mode == 'w-':
fid = h5f.create(name, h5f.ACC_EXCL, fapl=plist)
elif mode == 'w':
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=plist)
elif mode == 'a' or mode is None:
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=plist)
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=plist)
else:
raise ValueError("Invalid mode; must be one of r, r+, w, w-, a")
return fid
def make_fid(name, mode, userblock_size, fapl):
""" Get a new FileID by opening or creating a file.
Also validates mode argument."""
fcpl = None
if userblock_size is not None:
if mode in ('r', 'r+'):
raise ValueError(
"User block may only be specified when creating a file")
try:
userblock_size = int(userblock_size)
except (TypeError, ValueError):
raise ValueError("User block size must be an integer")
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_userblock(userblock_size)
if mode == 'r':
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
elif mode == 'r+':
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
elif mode == 'w-':
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode == 'w':
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
elif mode == 'a' or mode is None:
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
try:
existing_fcpl = fid.get_create_plist()
if userblock_size is not None and existing_fcpl.get_userblock(
) != userblock_size:
raise ValueError(
"Requested userblock size (%d) does not match that of existing file (%d)"
% (userblock_size, existing_fcpl.get_userblock()))
except:
fid.close()
raise
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
else:
raise ValueError("Invalid mode; must be one of r, r+, w, w-, a")
return fid
def make_fid(name, mode, userblock_size, fapl):
""" Get a new FileID by opening or creating a file.
Also validates mode argument."""
fcpl = None
if userblock_size is not None:
if mode in ('r', 'r+'):
raise ValueError("User block may only be specified "
"when creating a file")
try:
userblock_size = int(userblock_size)
except (TypeError, ValueError):
raise ValueError("User block size must be an integer")
fcpl = h5p.create(h5p.FILE_CREATE)
fcpl.set_userblock(userblock_size)
if mode == 'r':
fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl)
elif mode == 'r+':
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
elif mode == 'w-':
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
elif mode == 'w':
fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
elif mode == 'a' or mode is None:
try:
fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl)
try:
existing_fcpl = fid.get_create_plist()
if (userblock_size is not None and
existing_fcpl.get_userblock() != userblock_size):
raise ValueError("Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock()))
except:
fid.close()
raise
except IOError:
fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl)
else:
raise ValueError("Invalid mode; must be one of r, r+, w, w-, a")
return fid
def test_file(tmp_path):
image = decompress(a2b_base64(compressed_image))
# FAPL: File access property list
fapl = h5p.create(h5py.h5p.FILE_ACCESS)
fapl.set_fapl_core()
fapl.set_file_image(image)
file = tmp_path / "disk.h5"
fid = h5f.open(bytes(file), h5py.h5f.ACC_RDONLY, fapl=fapl)
with h5py.File(fid) as f:
assert (f["test"][:] == [1, 2, 3]).all()
assert f["test"].dtype == "int64"
def test_load_from_image(self):
from binascii import a2b_base64
from zlib import decompress
compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8='
image = decompress(a2b_base64(compressed_image))
fapl = h5p.create(h5py.h5p.FILE_ACCESS)
fapl.set_fapl_core()
fapl.set_file_image(image)
fid = h5f.open(self.mktemp().encode(), h5py.h5f.ACC_RDONLY, fapl=fapl)
f = h5py.File(fid)
self.assertTrue('test' in f)
def test_load_from_image(self):
from binascii import a2b_base64
from zlib import decompress
compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8='
image = decompress(a2b_base64(compressed_image))
fapl = h5p.create(h5py.h5p.FILE_ACCESS)
fapl.set_fapl_core()
fapl.set_file_image(image)
fid = h5f.open(self.mktemp().encode(), h5py.h5f.ACC_RDONLY, fapl=fapl)
f = h5py.File(fid)
self.assertTrue('test' in f)
def main():
sec2j.init()
fapl = h5p.create(h5p.FILE_ACCESS)
print fapl
print 'fapl.id:', fapl.id
sec2j.set_fapl(fapl.id)
fname = "test.h5"
if os.path.exists(fname):
print 'Opening the file...'
fid = h5f.open(fname, fapl=fapl)
else:
fid = h5f.create(fname, flags=h5f.ACC_TRUNC, fapl=fapl)
print 'fid ready:', fid.id
#return
f = h5py.File(fid)
sec2j.tx_start(fid.id)
g = f.require_group('bbic/volume/0')
for i in range(10000):
g.attrs.create('a%d' % i, 640)
f.flush()
os._exit(-1)
def setup_fid(self, name):
self.fname = res.get_data_copy(name)
plist = h5p.create(h5p.FILE_ACCESS)
plist.set_fclose_degree(h5f.CLOSE_STRONG)
self.fid = h5f.open(self.fname, h5f.ACC_RDWR, fapl=plist)
def setup_fid(self, name):
self.fname = res.get_data_copy(name)
plist = h5p.create(h5p.FILE_ACCESS)
plist.set_fclose_degree(h5f.CLOSE_STRONG)
self.fid = h5f.open(self.fname, h5f.ACC_RDWR, fapl=plist)
def visualizeGraph( fileStr, fileNrs, dotNameLinear, penWidthNodes=2, penWidthEdges=2, intensWindow=[], \
sortByIntensities=False ):
"""Use Graphviz for visualizing the lineage tree implicitly contained in the files
fileStr % fn (for all fn in fileNrs). The generated DOT instructions will be stored in the file
dotNameLinear (for the linear layout) and the total number of cells is returned.
Scaling factors for the pen width (line width) may be specified as well for nodes and edges.
Furthermore an intensity window can be provided that adjusts the fill colors of
nodes as well as the line width of edges: if intensWindow == [], all nodes are
printed the same way, but if intensWindow == [minIntens, maxIntens], the intensity
range [minIntens, maxIntens] is mapped to the value range of 0.2 - 1 in the HSV
color model. If sortByIntensities == True, the nodes are sorted by the average
intensities of the cells in descending order."""
minIndex = 0.05
if len(intensWindow) != 0:
if (len(intensWindow) != 2) or (intensWindow[0] >= intensWindow[1]):
raise RuntimeError("If intensWindow non-empty, it must contain two values (with the first " \
+ "being smaller")
outLin = open(dotNameLinear,'w')
print >> outLin, 'graph G {'
# we keep a dictionary of all currently living cells, which maps the integer number of this
# cell to the tuple consisting of the index of the node by which this cell is represented
# in the graph, plus the list of intensities from which the final pen width can be
# determined
livingCells = dict()
# We represent all nodes by a jagged array, with all nodes on the same level corresponding to a
# subarray. The first node is always the time line node. Newly appearing nodes are plotted in
# red, while disappearing nodes are marked with a cross.
fstr = fileStr % fileNrs[0]
fid = h5f.open( fstr, h5f.ACC_RDONLY )
print >> outLin, 'subgraph %d {\nrank=same;' % fileNrs[0]
# list of all edges which are added to the graph at the end
edgesLin = []
print >> outLin, '%d [color=white, shape=circle];' % fileNrs[0]
labcontent = readDataSet(fid, '/features/labelcontent', np.uint16)
cellIdx = 1
allCellStrs = []
for idx in range( len(labcontent) ):
if labcontent[idx]:
cellStr = 'C%d_%d' % (fileNrs[0], idx+1)
cii = getColorRankIndex(fid, idx+1, intensWindow, minIndex, False, \
sortByIntensities)
allCellStrs.append( (cellStr, cii[0], cii[1], cii[2]) )
livingCells[ idx+1 ] = (cellStr, [cii[2]])
cellIdx = cellIdx + 1
fid.close()
del fid
allCellStrs.sort(key=lambda x : x[2])
for cs in allCellStrs:
print >> outLin, cs[0] + (' [shape=circle, penwidth=%f, ' % (cs[3]*penWidthNodes)) + cs[1] + '];'
print >> outLin, '}'
fnPrev = fileNrs[0]
for fn in fileNrs[1:]:
print 'Processing file no. %d' % fn
print >> outLin, 'subgraph %d {\nrank=same;' % fn
print >> outLin, '%d [color=white, shape=circle];' % fn
edgesLin.append('%d -- %d [penwidth=%f];' % (fnPrev, fn, 0.1*penWidthEdges) )
fid = h5f.open(fileStr % fn, h5f.ACC_RDONLY)
gid = h5g.open(fid,'/tracking')
newLivingCells = dict()
allCellStrs = []
if 'Moves' in gid:
moves = readDataSet(gid, 'Moves')
nMoves = moves.shape[0]
for iM in range(nMoves):
newLivingCells[ moves[iM, 1] ] = livingCells[ moves[iM, 0] ]
cii = getColorRankIndex(fid, moves[iM, 1], intensWindow, minIndex, False, sortByIntensities)
newLivingCells[ moves[iM, 1] ][1].append(cii[2])
if 'Splits' in gid:
splits = readDataSet(gid, 'Splits')
nSplits = splits.shape[0]
for iS in range(nSplits):
(motherStr, motherInts) = livingCells[ splits[iS, 0] ]
cellStr1 = 'C%d_%d' % (fn, splits[iS, 1])
cellStr2 = 'C%d_%d' % (fn, splits[iS, 2])
cii1 = getColorRankIndex(fid, splits[iS, 1], intensWindow, minIndex, False, sortByIntensities)
cii2 = getColorRankIndex(fid, splits[iS, 2], intensWindow, minIndex, False, sortByIntensities)
allCellStrs.append( (cellStr1, cii1[0], cii1[1], cii1[2]) )
allCellStrs.append( (cellStr2, cii2[0], cii2[1], cii2[2]) )
newLivingCells[ splits[iS, 1] ] = (cellStr1, [cii1[2]])
ints1 = motherInts + [cii1[2]]
meanInt1 = reduce(operator.add, ints1) / len(ints1)
edgesLin.append(motherStr + ' -- ' + cellStr1 + ' [penwidth=%f];' % (meanInt1*penWidthEdges))
newLivingCells[ splits[iS, 2] ] = (cellStr2, [cii2[2]])
ints2 = motherInts + [cii2[2]]
meanInt2 = reduce(operator.add, ints2) / len(ints2)
edgesLin.append(motherStr + ' -- ' + cellStr2 + ' [penwidth=%f];' % (meanInt2*penWidthEdges))
cellIdx = cellIdx + 2
if 'Appearances' in gid:
apps = readDataSet(gid, 'Appearances')
nApps = apps.shape[0]
for iA in range(nApps):
cellStr = 'C%d_%d' % (fn, apps[iA])
cii = getColorRankIndex(fid, apps[iA], intensWindow, minIndex, True, sortByIntensities)
allCellStrs.append( (cellStr, cii[0], cii[1], cii[2]) )
newLivingCells[ apps[iA] ] = (cellStr, [cii[2]])
cellIdx = cellIdx + 1
if 'Disappearances' in gid:
disapps = readDataSet(gid, 'Disappearances')
nDisapps = disapps.shape[0]
for iDA in range(nDisapps):
cellStr = 'D%d_%d' % (fn, disapps[iDA])
print >> outLin, cellStr + ' [shape=point, penwidth=%f];' % (minIndex*penWidthNodes)
(motherStr, motherInts) = livingCells[ disapps[iDA] ]
meanInt = reduce(operator.add, motherInts) / len(motherInts)
edgesLin.append(motherStr + ' -- ' + cellStr + ' [penwidth=%f];' % (meanInt*penWidthEdges) )
cellIdx = cellIdx + 1
fid.close()
del fid
for cs in allCellStrs:
print >> outLin, cs[0] + (' [shape=circle, penwidth=%f, ' % (cs[3]*penWidthNodes)) + cs[1] + '];'
print >> outLin, '}\n'
livingCells = newLivingCells
fnPrev = fn
print >> outLin, 'subgraph end {\nrank=same;'
for k in livingCells.keys():
cellStr = 'L%d' % k
print >> outLin, cellStr + ' [shape=none, penwidth=%f];' % (2*minIndex*penWidthNodes)
(motherStr, motherInts) = livingCells[k]
meanInt = reduce(operator.add, motherInts)/len(motherInts)
edgesLin.append(motherStr + ' -- ' + cellStr + ' [penwidth=%f];' % (0.2*penWidthEdges) )
print >> outLin, '}\n'
for e in edgesLin:
print >> outLin, e
print >> outLin, '}'
outLin.close()
return (cellIdx-1)
