def create_dl0_table_tel(hfile, telNode, nbGain, nbPixel, nbSlice, chunkshape=1):
"""
Create the waveform tables into the given telescope node
Parameters:
hfile : HDF5 file to be used
telNode : telescope to be completed
nbGain : number of gains of the camera
nbPixel : number of pixels
nbSlice : number of slices
chunkshape : shape of the chunk to be used to store the data
"""
if nbGain > 1:
pixelLo = hfile.create_vlarray(telNode, "pixelLo", tables.UInt16Atom(shape=()), "table of the index of the pixels which are in low gain mode",)
pixelWaveform = hfile.create_vlarray(telNode, "pixelWaveform", tables.UInt16Atom(shape=()), "table of the index of the pixels recorded with the waveform",)
#columns_dict_waveformoffset = {"waveformoffset": tables.UInt64Col(shape=())}
#description_waveformoffset = type('description columns_dict_waveformoffset', (tables.IsDescription,), columns_dict_waveformoffset)
#hfile.create_table(telNode, 'waveformoffset', description_waveformoffset, "Offset of the waveform stored in the waveform table", chunkshape=chunkshape)
columns_dict_waveform = {"waveform": tables.UInt16Col(shape=nbSlice)}
description_waveform = type('description columns_dict_waveform', (tables.IsDescription,), columns_dict_waveform)
hfile.create_table(telNode, 'waveform', description_waveform, "Table of waveform of the pixel with waveform", chunkshape=chunkshape)
columns_dict_signal = {
#"signal": tables.Float32Col(shape=(nbPixel)),
"signal": tables.Int16Col(shape=(nbPixel)),
"waveformoffset": tables.UInt64Col(shape=())
}
description_signal = type('description columns_dict_signal', (tables.IsDescription,), columns_dict_signal)
hfile.create_table(telNode, 'signal', description_signal, "Calibrated and integrated signal", chunkshape=chunkshape)
def __init__(self, h5_name):
self.hdf5_name = '%s.h5' % h5_name
try:
os.makedirs(cache_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
self.hdf5_path = os.path.join(cache_dir, self.hdf5_name)
self.hdf5_write_mode = 'a' # default to append mode, can be changed to 'w' mode using overwrite
self.train_unalt_set = TrainingSet(manip=False)
self.train_manip_set = TrainingSet(manip=True)
self.test_unalt_set = TestSet(manip=False)
self.test_manip_set = TestSet(manip=True)
self.data_src = {
'train_unalt': self.train_unalt_set,
'train_manip': self.train_manip_set,
'test_unalt': self.test_unalt_set,
'test_manip': self.test_manip_set
}
print('# train images: %d\n'
'# train_manip images: %d\n'
'# test images: %d\n'
'# test_manip images: %d\n' % (self.train_unalt_set.n_files,
self.train_manip_set.n_files,
self.test_unalt_set.n_files,
self.test_manip_set.n_files))
self.feature_dtype = tables.Float32Atom()
self.data_type = {
'x': tables.UInt8Atom(),
'y': tables.UInt8Atom(),
'image_index': tables.UInt16Atom(),
'patch_coord': tables.UInt16Atom(),
'manip': tables.UInt8Atom(),
}
self.data_shape = {
'y': (0,),
'patch_coord': (0, 2),
'image_index': (0, ),
'manip': (0, )
}
self.filters = tables.Filters(complevel=5, complib='blosc')
self.label_shape = (0,)
self.coord_shape = (0, 2)
self.index_shape = (0, )
self.manip_shape = (0, )
def __init__(self, path: str,
event_table_desc: Type[tables.IsDescription]):
"""
Writer for a collection of Photoelectrons to HDF5 file using pytables
Stores the photoelectron info as variable length arrays
Parameters
----------
path : str
Path to store the file (overwrites if already exists)
event_table_desc : Type[tables.IsDescription]
Uninstanced `tables.IsDescription` class describing the columns of
the event metadata table
"""
self._file = tables.File(path, mode='w', filters=FILTERS)
group = self._file.create_group(self._file.root, "data", "Event Data")
self._event_metadata_table = self._file.create_table(
group, "event_metadata", event_table_desc, "Event Metadata")
self._pixel_column = self._file.create_vlarray(
group, "photoelectron_arrival_pixel", tables.UInt16Atom(shape=()),
"Pixel hit by the photoelectron")
self._time_column = self._file.create_vlarray(
group, "photoelectron_arrival_time", tables.Float64Atom(shape=()),
"Arrival time of the photoelectrons")
self._charge_column = self._file.create_vlarray(
group, "photoelectron_measured_charge",
tables.Float64Atom(shape=()),
"Charge reported by photosensor for each photoelectron")
def from_array(self, data, labels):
"""Fill matrix from 2 dimensional array
Args:
data (np.array): 2 dimensional square array with scores
labels (list): List of labels for each column and row index
"""
labels = [np.string_(d) for d in labels]
self.labels = self.h5file.create_carray('/',
'labels',
obj=labels,
filters=self.filters)
self.h5file.flush()
self.build_label_cache()
nr_frags = len(labels)
self.scores = self.h5file.create_carray('/',
'scores',
atom=tables.UInt16Atom(),
shape=(nr_frags, nr_frags),
chunkshape=(1, nr_frags),
filters=self.filters)
self.scores[0:nr_frags,
0:nr_frags] = (data *
self.score_precision).astype('uint16')
def convertFiles(pathToData, outFile, complib='zlib', complevel=9):
seriesName = pathToData.split(os.sep)[-2]
fnl = os.listdir(pathToData)
fnl.sort()
fnl2 = [pathToData + f for f in fnl]
#f1 = read_kdf.ReadKdfData(fnl2[0])
f1 = array(Image.open(fnl2[0])).newbyteorder('BE')
xSize, ySize = f1.shape[0:2]
outF = tables.open_file(outFile, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
imageData = outF.create_earray(outF.root, 'ImageData', tables.UInt16Atom(), (0,xSize,ySize), filters=filt, expectedrows=len(fnl2))
for fn in fnl2:
#print fn
imageData.append(array(Image.open(fn)).newbyteorder('BE').reshape(1, xSize, ySize))
outEvents = outF.create_table(outF.root, 'Events', SpoolEvent,filters=tables.Filters(complevel=5, shuffle=True))
outF.flush()
outF.close()
def __init__(self, output_dir, chrom_list):
# combined allele-specific read counts
as_count_filename = "%s/combined_as_count.h5" % output_dir
self.as_count_h5 = tables.openFile(as_count_filename, "w")
# combined mapped read counts
read_count_filename = "%s/combined_read_count.h5" % output_dir
self.read_count_h5 = tables.openFile(read_count_filename, "w")
# counts of genotypes
ref_count_filename = "%s/combined_ref_count.h5" % output_dir
self.ref_count_h5 = tables.openFile(ref_count_filename, "w")
alt_count_filename = "%s/combined_alt_count.h5" % output_dir
self.alt_count_h5 = tables.openFile(alt_count_filename, "w")
het_count_filename = "%s/combined_het_count.h5" % output_dir
self.het_count_h5 = tables.openFile(het_count_filename, "w")
self.filenames = [
as_count_filename, read_count_filename, ref_count_filename,
alt_count_filename, het_count_filename
]
self.h5_files = [
self.as_count_h5, self.read_count_h5, self.ref_count_h5,
self.alt_count_h5, self.het_count_h5
]
# initialize all of these files
atom = tables.UInt16Atom(dflt=0)
for h5f in self.h5_files:
for chrom in chrom_list:
self.create_carray(h5f, chrom, atom)
def _checkCreateDataTable(self, f):
if (not 'ImageData' in dir(self.h5f.root)) and (not 'PZFImageData'
in dir(self.h5f.root)):
if isinstance(f, str):
#is a PZF file
f = PZFFormat.loads(f)[0]
f.reshape((1, ) + f.shape[:2])
framesize = f.shape[1:3]
self.dshape[:2] = framesize
if not self.usePZFFormat:
filt = tables.Filters(self.complevel,
self.complib,
shuffle=True)
self.imageData = self.h5f.create_earray(
self.h5f.root,
'ImageData',
tables.UInt16Atom(), (0, ) + tuple(framesize),
filters=filt,
chunkshape=(1, ) + tuple(framesize))
else:
self.compImageData = self.h5f.create_vlarray(
self.h5f.root, 'PZFImageData', tables.VLStringAtom())
self.compImageData.attrs.framesize = framesize
def add_event_type(fd, id=None, evt=None):
"""fd is returned by `open_files`: it is a dict {type: tb_file_handle}."""
kwik = fd.get('kwik', None)
# The KWIK needs to be there.
assert kwik is not None
if id is None:
# If id is None, take the maximum integer index among the existing
# recording names, + 1.
event_types = sorted([n._v_name
for n in kwik.listNodes('/event_types')])
if event_types:
id = str(max([int(r) for r in event_types if r.isdigit()]) + 1)
else:
id = '0'
event_type = kwik.createGroup('/event_types', id)
kwik.createGroup(event_type, 'user_data')
app = kwik.createGroup(event_type, 'application_data')
kv = kwik.createGroup(app, 'klustaviewa')
kv._f_setAttr('color', None)
events = kwik.createGroup(event_type, 'events')
kwik.createEArray(events, 'time_samples', tb.UInt64Atom(), (0,))
kwik.createEArray(events, 'recording', tb.UInt16Atom(), (0,))
kwik.createGroup(events, 'user_data')
def convertFiles(pathToData, outFile, complib='zlib', complevel=9):
seriesName = pathToData.split(os.sep)[-2]
fnl = os.listdir(pathToData)
fnl2 = [pathToData + f for f in fnl]
f1 = read_kdf.ReadKdfData(fnl2[0])
xSize, ySize = f1.shape[0:2]
outF = tables.open_file(outFile, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
imageData = outF.create_earray(outF.root,
'ImageData',
tables.UInt16Atom(), (0, xSize, ySize),
filters=filt,
expectedrows=len(fnl2))
for fn in fnl2:
imageData.append(read_kdf.ReadKdfData(fn).reshape(1, xSize, ySize))
outF.flush()
outF.close()
def dump_test_set(self, h5filepath, nframes, framesize):
# set rng to a hardcoded state, so we always have the same test set!
self.numpy_rng.seed(1)
with tables.openFile(h5filepath, 'w') as h5file:
h5file.createArray(h5file.root, 'test_targets',
self.partitions['test']['targets'])
vids = h5file.createCArray(h5file.root,
'test_images',
tables.Float32Atom(),
shape=(10000, nframes, framesize,
framesize),
filters=tables.Filters(complevel=5,
complib='zlib'))
pos = h5file.createCArray(h5file.root,
'test_pos',
tables.UInt16Atom(),
shape=(10000, nframes, 2),
filters=tables.Filters(complevel=5,
complib='zlib'))
for i in range(100):
print i
(vids[i * 100:(i + 1) * 100], pos[i * 100:(i + 1) * 100],
_) = self.get_batch('test',
100,
nframes,
framesize,
idx=np.arange(i * 100, (i + 1) * 100))
h5file.flush()
def convertFile(pathToData,
outFile,
frameSize=[256, 256],
pixelsize=None,
complib='zlib',
complevel=9):
#seriesName = pathToData.split(os.sep)[-2]
#fnl = os.listdir(pathToData)
#fnl2 = [pathToData + f for f in fnl]
#f1 = read_kdf.ReadKdfData(pathToData).squeeze()
xSize, ySize = frameSize
f1 = open(pathToData, 'rb')
#detect file length
f1.seek(0, 2) #seek to end
fLength = f1.tell()
f1.seek(0) #back to begining
nFrames = fLength / (2 * xSize * ySize)
outF = tables.openFile(outFile, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
imageData = outF.createEArray(outF.root,
'ImageData',
tables.UInt16Atom(), (0, xSize, ySize),
filters=filt,
expectedrows=nFrames)
for i in range(nFrames):
d1 = numpy.fromfile(f1, '>u2', xSize * ySize) >> 4
imageData.append(d1.reshape(1, xSize, ySize))
if i % 100 == 0:
print(('%d of %d frames' % (i, nFrames)))
f1.close()
hdh = MetaData.HDFMDHandler(outF, MetaData.TIRFDefault)
if not pixelsize == None:
hdh.setEntry('voxelsize.x', pixelsize)
hdh.setEntry('voxelsize.y', pixelsize)
outEvents = outF.createTable(outF.root,
'Events',
SpoolEvent,
filters=tables.Filters(complevel=5,
shuffle=True))
outF.flush()
outF.close()
def __init__(self, scope, filename, acquisator, protocol = p.NullProtocol, parent=None, complevel=6, complib='zlib'):
self.h5File = tables.openFile(filename, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
self.imageData = self.h5File.createEArray(self.h5File.root, 'ImageData', tables.UInt16Atom(), (0,scope.cam.GetPicWidth(),scope.cam.GetPicHeight()), filters=filt)
self.md = MetaDataHandler.HDFMDHandler(self.h5File)
self.evtLogger = EventLogger(self, scope, self.h5File)
sp.Spooler.__init__(self, scope, filename, acquisator, protocol, parent)
def save(fname, I):
fid = tables.openFile(fname, mode='w')
try:
atom = tables.UInt16Atom()
filter = tables.Filters(complevel=9, complib='zlib')
A = fid.createCArray(fid.root, 'I', atom, I.shape, filters=filter)
A[:] = I
fid.close()
except:
print 'Error saving file.'
fid.close()
def create_carray(track, chrom):
atom = tables.UInt16Atom(dflt=0)
zlib_filter = tables.Filters(complevel=1, complib="zlib")
# create CArray for this chromosome
shape = [chrom.length]
carray = track.h5f.createCArray(track.h5f.root, chrom.name,
atom, shape, filters=zlib_filter)
return carray
def extractFrames(dataSource,
metadata,
origName,
outFile,
start,
end,
subsamp=1,
complib='zlib',
complevel=5):
h5out = tables.openFile(outFile, 'w')
filters = tables.Filters(complevel, complib, shuffle=True)
nframes = end - start
xSize, ySize = dataSource.getSliceShape()
ims = h5out.createEArray(h5out.root,
'ImageData',
tables.UInt16Atom(), (0, xSize, ySize),
filters=filters,
expectedrows=nframes)
for frameN in range(start, end, subsamp):
im = dataSource.getSlice(frameN)[None, :, :]
for fN in range(frameN + 1, frameN + subsamp):
im += dataSource.getSlice(fN)[None, :, :]
ims.append(im)
ims.flush()
outMDH = MetaDataHandler.HDFMDHandler(h5out)
outMDH.copyEntriesFrom(metadata)
outMDH.setEntry('cropping.originalFile', origName)
outMDH.setEntry('cropping.start', start)
outMDH.setEntry('cropping.end', end)
outMDH.setEntry('cropping.averaging', subsamp)
if 'Camera.ADOffset' in metadata.getEntryNames():
outMDH.setEntry('Camera.ADOffset',
subsamp * metadata.getEntry('Camera.ADOffset'))
outEvents = h5out.createTable(h5out.root,
'Events',
SpoolEvent,
filters=tables.Filters(complevel=5,
shuffle=True))
#copy events to results file
evts = dataSource.getEvents()
if len(evts) > 0:
outEvents.append(evts)
h5out.flush()
h5out.close()
def create_event_subarray_trigger(hfile, event_subarray_group):
"""
Create the event subarray trigger table
Parameters:
hfile: HDF5 file to be used
event_subarray_group:
"""
hfile.create_table(event_subarray_group, 'trigger', EventSubarrayTrigger,
'Trigger information')
hfile.create_vlarray(event_subarray_group, "tels_with_trigger",
tables.UInt16Atom(shape=()),
'Telescope that have triggered - tels_with_data')
def _checkCreateDataTable(self, f):
if not 'ImageData' in dir(self.h5f.root):
filt = tables.Filters(self.complevel, self.complib, shuffle=True)
framesize = f.shape[1:3]
self.imageData = self.h5f.createEArray(
self.h5f.root,
'ImageData',
tables.UInt16Atom(), (0, ) + tuple(framesize),
filters=filt,
chunkshape=(1, ) + tuple(framesize))
#self.events = self.h5DataFile.createTable(self.h5DataFile.root, 'Events', HDFTaskQueue.SpoolEvent,filters=filt)
self.dshape[:2] = framesize
def create_gc_DenseTrackSet(outTableFile, contigLengths, grp, overwrite_b):
print "creating GC Dense Track Set..."
GC = DenseTrackSet(contigLengths,
outTableFile,
overwrite_b,
'w',
compression=True)
GC.addGroup(grp)
GC[grp].addArray(tables.UInt16Atom(), [])
print "done"
return GC
def saveSequence(self, fileName, frames, times=None, filterLevel=5):
if frames is None:
return
if fileName is None:
return
h, w, nFrames = frames.shape
h5file = tb.openFile(fileName, mode='w')
root = h5file.root
atom = tb.UInt16Atom() #tb.Atom.from_dtype(data.dtype)
if filterLevel == 0:
filters = None
else:
filters = tb.Filters(complevel=filterLevel,
complib='zlib',
shuffle=True)
x = h5file.createEArray(root,
'x',
atom,
shape=(h, w, 0),
expectedrows=nFrames,
filters=filters)
for i in range(nFrames):
data = frames[:, :, i]
try:
x.append(data)
except ValueError:
print("Can't add frame " + str(i) + ". Wrong dimensions?")
if times != None:
tGroup = h5file.createGroup(root, 'times')
tA = np.vstack((range(1, nFrames + 1), times)).T
_t = h5file.createCArray(tGroup,
'timesArray',
tb.Atom.from_dtype(tA.dtype),
tA.shape,
filters=filters)
_t[:] = tA
bytes = np.fromstring(self.timesLabel.encode('utf-8'), np.uint8)
_tStr = h5file.createCArray(tGroup,
'timeLabel',
tb.UInt8Atom(),
shape=(len(bytes), ),
filters=filters)
_tStr[:] = bytes
h5file.flush()
h5file.close()
def preprocess_images(filedir, filenames, lateralities, datafilename):
ten_percent = int(round(len(filenames) / 10))
processname = multiprocessing.current_process().name
datafile = tables.open_file(datafilename, mode='w')
data = datafile.create_earray(datafile.root, 'data', tables.UInt16Atom(shape=EXPECTED_DIM), (0,), 'dream')
total = len(filenames)
count = 0
for i in range(len(filenames)):
data.append(preprocess_image(os.path.join(filedir, filenames[i]), lateralities[i]))
count += 1
if count >= ten_percent and count % ten_percent == 0:
print('{}: {}/{}'.format(processname, count, total))
print('{}: {}/{}'.format(processname, count, total))
datafile.close()
def _create_table_list(self, name, example):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
The modified version for creating table with appendList
"""
type_map = {
np.dtype(np.float64): tables.Float64Atom(),
np.dtype(np.float32): tables.Float32Atom(),
np.dtype(np.int): tables.Int64Atom(),
np.dtype(np.int8): tables.Int8Atom(),
np.dtype(np.uint8): tables.UInt8Atom(),
np.dtype(np.int16): tables.Int16Atom(),
np.dtype(np.uint16): tables.UInt16Atom(),
np.dtype(np.int32): tables.Int32Atom(),
np.dtype(np.uint32): tables.UInt32Atom(),
np.dtype(np.bool): tables.BoolAtom(),
}
try:
if type(example) == np.ndarray:
h5type = type_map[example.dtype]
elif type(example) == list and type(example[0]) == str:
h5type = tables.VLStringAtom()
except KeyError:
raise TypeError("Don't know how to handle dtype '%s'" %
example.dtype)
if type(example) == np.ndarray:
h5dim = (0, ) + example.shape[1:]
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_earray(h5.root,
name,
h5type,
h5dim,
filters=filters)
elif type(example) == list and type(example[0]) == str:
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_vlarray(h5.root,
name,
h5type,
filters=filters)
self.types[name] = type(example)
def _create_table(self, name, example):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
"""
type_map = {
np.dtype(np.float64): tables.Float64Atom(),
np.dtype(np.float32): tables.Float32Atom(),
np.dtype(np.int): tables.Int64Atom(),
np.dtype(np.int8): tables.Int8Atom(),
np.dtype(np.uint8): tables.UInt8Atom(),
np.dtype(np.int16): tables.Int16Atom(),
np.dtype(np.uint16): tables.UInt16Atom(),
np.dtype(np.int32): tables.Int32Atom(),
np.dtype(np.uint32): tables.UInt32Atom(),
np.dtype(np.bool): tables.BoolAtom(),
}
try:
if type(example) == np.ndarray:
h5type = type_map[example.dtype]
elif type(example) == str:
h5type = tables.VLStringAtom()
except KeyError:
raise TypeError(
"Could not create table %s because of unknown dtype '%s'" %
(name, example.dtype)) #+ ", of name: " % example.shape)
if type(example) == np.ndarray:
h5dim = (0, ) + example.shape
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_earray(h5.root,
name,
h5type,
h5dim,
filters=filters)
elif type(example) == str:
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
self.tables[name] = h5.create_vlarray(h5.root,
name,
h5type,
filters=filters)
self.types[name] = type(example)
def slicer_to_pytable(input_file, save_path, desamp=1):
mov = BinarySlicer(input_file)
t = mov.shape[0]
y = mov.shape[1]
x = mov.shape[2]
basepath, filename = os.path.split(input_file)
base_save_path, save_name = os.path.split(save_path)
final_size = y / desamp
output_name = filename[:-4] + '_' + str(final_size) + 'x' + str(
final_size) + '_' + save_name + '.h5'
output_path = os.path.join(base_save_path, output_name)
# Create output file using a pytables compression array:
f = tb.open_file(output_path, mode="w", title="mov_h5")
mov_h5 = f.create_carray(f.root,
'data',
tb.UInt16Atom(),
shape=(t, y / desamp, x / desamp))
print 'saveing to ' + str(output_path)
if desamp == 1:
start_time = timeit.default_timer()
for i in range(t):
mov_h5[i, :, :] = mov[i]
f.close()
run_time = timeit.default_timer() - start_time
print '.npy to .h5 conversion took ' + str(run_time) + ' seconds'
else:
#Decimate by averaging:
start_time = timeit.default_timer()
for i in range(t):
mov_h5[i, :, :] = block_reduce(mov[i],
block_size=(desamp, desamp),
func=np.mean)
f.close()
run_time = timeit.default_timer() - start_time
print 'conversion and decimation took ' + str(run_time) + ' seconds'
return output_path
def create_carray(h5f, chrom, data_type):
if data_type == "uint8":
atom = tables.UInt8Atom(dflt=0)
elif data_type == "uint16":
atom = tables.UInt16Atom(dflt=0)
else:
raise NotImplementedError("unsupported datatype %s" % data_type)
zlib_filter = tables.Filters(complevel=1, complib="zlib")
# create CArray for this chromosome
shape = [chrom.length]
carray = h5f.create_carray(h5f.root, chrom.name,
atom, shape, filters=zlib_filter)
return carray
def _clean_hdf5_movie(fname_ori, fname_new, model, scale_log):
TABLE_FILTERS = tables.Filters(complevel=5,
complib='blosc',
shuffle=True,
fletcher32=True)
reader = MovieReader(str(fname_ori))
tot = len(reader)
height = reader.height
width = reader.width
with tables.File(str(fname_new), 'w') as fid_new:
block = fid_new.create_carray(
'/',
'mask',
#tables.Float32Atom(),
tables.UInt16Atom(),
shape=(tot, height, width),
chunkshape=(1, height, width),
filters=TABLE_FILTERS)
batch = []
for ii, (header, img) in enumerate(tqdm.tqdm(reader)):
batch.append(img[None].astype(np.float32))
if len(batch) >= batch_size or ii == (tot - 1):
with torch.no_grad():
Xv = torch.from_numpy(np.array(batch))
batch = []
Xv = Xv.to(device)
Xv.log_().add_(-scale_log[0]).div_(scale_log[1] -
scale_log[0])
Xhat = model(Xv)
Xhat.mul_(scale_log[1] - scale_log[0]).add_(
scale_log[0]).exp_()
ss = Xhat.shape
xhat = Xhat.cpu().detach().view(ss[0], ss[2],
ss[3]).numpy()
xhat = xhat.round().clip(0, 2**16 - 1).astype(np.uint16)
block[ii - xhat.shape[0] + 1:ii + 1] = xhat
def _create_table(self, name, example, parent=None):
"""
Create a new table within the HDF file, where the tables shape and its
datatype are determined by *example*.
"""
h5 = self.h5
filters = tables.Filters(complevel=self.compression_level,
complib='zlib',
shuffle=True)
if parent is None:
parent = h5.root
if type(example) == str:
h5type = tables.VLStringAtom()
h5.createVLArray(parent, name, h5type, filters=filters)
return
if type(example) == dict:
self.h5.createGroup(parent, name)
return
#If we get here then we're dealing with numpy arrays
example = np.asarray(example)
#MODIFICATION: appended name everywhere and introduced string
type_map = {
np.dtype(np.float64).name: tables.Float64Atom(),
np.dtype(np.float32).name: tables.Float32Atom(),
np.dtype(np.int).name: tables.Int64Atom(),
np.dtype(np.int8).name: tables.Int8Atom(),
np.dtype(np.uint8).name: tables.UInt8Atom(),
np.dtype(np.int16).name: tables.Int16Atom(),
np.dtype(np.uint16).name: tables.UInt16Atom(),
np.dtype(np.int32).name: tables.Int32Atom(),
np.dtype(np.uint32).name: tables.UInt32Atom(),
np.dtype(np.bool).name: tables.BoolAtom(),
# Maximal string length of 128 per string - change if needed
'string32': tables.StringAtom(128)
}
try:
h5type = type_map[example.dtype.name]
h5dim = (0, ) + example.shape
h5.createEArray(parent, name, h5type, h5dim, filters=filters)
except KeyError:
raise TypeError("Don't know how to handle dtype '%s'" %
example.dtype)
def __init__(self,
fnContigLengths,
fnWssd,
overwrite,
openMode,
tracksetsToCheck=[],
compression=False,
datatype=tables.UInt16Atom()):
DenseTrackSet.__init__(self,
fnContigLengths,
fnWssd,
overwrite,
openMode,
groupsToCheck=tracksetsToCheck,
compression=compression)
self.datatype = datatype
self.depthgetter = WssdFile.DepthGetter(self)
def convertFile(pathToData,
outFile,
pixelsize=None,
complib='zlib',
complevel=9):
#seriesName = pathToData.split(os.sep)[-2]
#fnl = os.listdir(pathToData)
#fnl2 = [pathToData + f for f in fnl]
f1 = read_kdf.ReadKdfData(pathToData).squeeze()
xSize, ySize = f1.shape[0:2]
outF = tables.openFile(outFile, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
imageData = outF.createEArray(outF.root,
'ImageData',
tables.UInt16Atom(), (0, xSize, ySize),
filters=filt,
expectedrows=f1.shape[2])
for i in range(f1.shape[2]):
imageData.append(f1[:, :, i].reshape(1, xSize, ySize))
hdh = MetaData.HDFMDHandler(outF, MetaData.PCODefault)
if not pixelsize == None:
hdh.setEntry('voxelsize.x', pixelsize)
hdh.setEntry('voxelsize.y', pixelsize)
outEvents = outF.createTable(outF.root,
'Events',
SpoolEvent,
filters=tables.Filters(complevel=5,
shuffle=True))
outF.flush()
outF.close()
def __init__(self,
filename,
frameSource,
frameShape,
complevel=6,
complib='zlib',
**kwargs):
self.h5File = tables.open_file(filename, 'w')
filt = tables.Filters(complevel, complib, shuffle=True)
self.imageData = self.h5File.create_earray(
self.h5File.root,
'ImageData',
tables.UInt16Atom(), (0, frameShape[0], frameShape[1]),
filters=filt)
self.md = MetaDataHandler.HDFMDHandler(self.h5File)
self.evtLogger = EventLogger(self, self.h5File)
sp.Spooler.__init__(self, filename, frameSource, **kwargs)
def create_carray(track, chrom, dtype):
if dtype == 'uint8':
atom = tables.UInt8Atom(dflt=0)
elif dtype == 'uint16':
atom = tables.UInt16Atom(dflt=0)
else:
raise NotImplementedError("support for dtype %s not "
"yet implemented" % dtype)
zlib_filter = tables.Filters(complevel=1, complib="zlib")
# create CArray for this chromosome
shape = [chrom.length]
carray = track.h5f.createCArray(track.h5f.root,
chrom.name,
atom,
shape,
filters=zlib_filter)
return carray