def _do_initialize(self):
header = self._header = _read_header(self._path, HEADER_FIELDS)
self._image_offset = _get_image_offset(header)
if header['version'] >= 5: # StreamPix version 6
# Timestamp = 4-byte unsigned long + 2-byte unsigned short (ms)
# + 2-byte unsigned short (us)
self._timestamp_micro = True
else: # Older versions
self._timestamp_micro = False
try:
dtype = np.dtype('uint%i' % header['bit_depth'])
except TypeError:
raise DataSetException("unsupported bit depth: %s" %
header['bit_depth'])
frame_size_bytes = header['width'] * header['height'] * dtype.itemsize
self._footer_size = header['true_image_size'] - frame_size_bytes
self._filesize = os.stat(self._path).st_size
self._image_count = int(
(self._filesize - self._image_offset) / header['true_image_size'])
if self._image_count != np.prod(self._scan_size):
raise DataSetException("scan_size doesn't match number of frames")
shape = Shape(
self._scan_size + (header['height'], header['width']),
sig_dims=2,
)
self._meta = DataSetMeta(
shape=shape,
raw_dtype=dtype,
dtype=dtype,
metadata=header,
)
self._maybe_load_dark_gain()
return self
def __init__(self,
tileshape=None,
num_partitions=None,
data=None,
sig_dims=2,
check_cast=True,
crop_frames=False,
tiledelay=None,
datashape=None):
# For HTTP API testing purposes: Allow to create empty dataset with given shape
if data is None:
assert datashape is not None
data = np.zeros(datashape, dtype=np.float32)
if num_partitions is None:
num_partitions = psutil.cpu_count(logical=False)
if tileshape is None:
sig_shape = data.shape[-sig_dims:]
target = 2**20
framesize = np.prod(sig_shape)
framecount = max(
1, min(np.prod(data.shape[:-sig_dims]),
int(target / framesize)))
tileshape = (framecount, ) + sig_shape
assert len(tileshape) == sig_dims + 1
self.data = data
self.tileshape = Shape(tileshape, sig_dims=sig_dims)
self.num_partitions = num_partitions
self.sig_dims = sig_dims
self._meta = DataSetMeta(
shape=self.shape,
raw_dtype=self.data.dtype,
)
self._check_cast = check_cast
self._crop_frames = crop_frames
self._tiledelay = tiledelay
def initialize(self, executor):
# FIXME: possibly need to have an "acquisition plan" object
# so we know all relevant parameters beforehand
dtype = np.uint8 # FIXME: don't know the dtype yet
self._meta = DataSetMeta(
shape=Shape(self._nav_shape + self._sig_shape, sig_dims=2),
raw_dtype=dtype,
dtype=dtype,
)
return self
def __init__(self, data, tileshape, num_partitions, sig_dims=2):
assert len(tileshape) == sig_dims + 1
self.data = data
self.tileshape = Shape(tileshape, sig_dims=sig_dims)
self.num_partitions = num_partitions
self.sig_dims = sig_dims
self._meta = DataSetMeta(
shape=self.shape,
raw_dtype=self.data.dtype,
)
def __init__(self, tileshape=None, num_partitions=None, data=None, sig_dims=2,
check_cast=True, tiledelay=None, datashape=None, base_shape=None,
force_need_decode=False, io_backend=None,
nav_shape=None, sig_shape=None, sync_offset=0):
super().__init__(io_backend=io_backend)
if io_backend is not None:
raise ValueError("MemoryDataSet currently doesn't support alternative I/O backends")
# For HTTP API testing purposes: Allow to create empty dataset with given shape
if data is None:
assert datashape is not None
data = np.zeros(datashape, dtype=np.float32)
if num_partitions is None:
num_partitions = psutil.cpu_count(logical=False)
# if tileshape is None:
# sig_shape = data.shape[-sig_dims:]
# target = 2**20
# framesize = np.prod(sig_shape)
# framecount = max(1, min(np.prod(data.shape[:-sig_dims]), int(target / framesize)))
# tileshape = (framecount, ) + sig_shape
self.data = data
self._base_shape = base_shape
self.num_partitions = num_partitions
self.sig_dims = sig_dims
if nav_shape is None:
self._nav_shape = self.shape.nav
else:
self._nav_shape = tuple(nav_shape)
if sig_shape is None:
self._sig_shape = self.shape.sig
else:
self._sig_shape = tuple(sig_shape)
self.sig_dims = len(self._sig_shape)
self._sync_offset = sync_offset
self._check_cast = check_cast
self._tiledelay = tiledelay
self._force_need_decode = force_need_decode
self._image_count = int(np.prod(self._nav_shape))
self._shape = Shape(
tuple(self._nav_shape) + tuple(self._sig_shape), sig_dims=self.sig_dims
)
if tileshape is None:
self.tileshape = None
else:
assert len(tileshape) == self.sig_dims + 1
self.tileshape = Shape(tileshape, sig_dims=self.sig_dims)
self._nav_shape_product = int(np.prod(self._nav_shape))
self._sync_offset_info = self.get_sync_offset_info()
self._meta = DataSetMeta(
shape=self._shape,
raw_dtype=self.data.dtype,
sync_offset=self._sync_offset,
image_count=self._image_count,
)
def initialize(self, executor):
self._array = self._adapt_chunking(self._array, self._sig_dims)
self._nav_shape = self._array.shape[:-self._sig_dims]
self._nav_shape_product = int(np.prod(self._nav_shape))
self._image_count = self._nav_shape_product
shape = Shape(self._nav_shape + self._sig_shape, sig_dims=self._sig_dims)
self._meta = DataSetMeta(
shape=shape,
raw_dtype=np.dtype(self._dtype),
sync_offset=0,
image_count=self._nav_shape_product,
)
return self
def wait_for_acquisition(self):
logger.info("waiting for acquisition header")
header = self._data_source.socket.get_acquisition_header()
bit_depth = int(header['Counter Depth (number)'])
if bit_depth in (1, 6):
dtype = np.uint8
elif bit_depth in (12, ):
dtype = np.uint16
else: # 24 bit?
dtype = np.uint32
self._meta = DataSetMeta(
shape=Shape(self._scan_size + (256, 256), sig_dims=2),
raw_dtype=dtype,
dtype=dtype,
)
return header
def __init__(self,
data,
tileshape,
partition_shape,
sig_dims=2,
effective_shape=None):
self.data = data
self.tileshape = Shape(tileshape, sig_dims=sig_dims)
self.partition_shape = Shape(partition_shape, sig_dims=sig_dims)
self.sig_dims = sig_dims
self._effective_shape = effective_shape and Shape(
effective_shape, sig_dims) or None
self._meta = DataSetMeta(
shape=self.shape,
raw_shape=self.raw_shape,
dtype=self.data.dtype,
)
def _do_initialize(self):
header = self._header = _read_header(self._path, HEADER_FIELDS)
self._image_offset = _get_image_offset(header)
if header['version'] >= 5: # StreamPix version 6
# Timestamp = 4-byte unsigned long + 2-byte unsigned short (ms)
# + 2-byte unsigned short (us)
self._timestamp_micro = True
else: # Older versions
self._timestamp_micro = False
try:
dtype = np.dtype('uint%i' % header['bit_depth'])
except TypeError:
raise DataSetException("unsupported bit depth: %s" %
header['bit_depth'])
frame_size_bytes = header['width'] * header['height'] * dtype.itemsize
self._footer_size = header['true_image_size'] - frame_size_bytes
self._filesize = os.stat(self._path).st_size
self._image_count = int(
(self._filesize - self._image_offset) / header['true_image_size'])
if self._sig_shape is None:
self._sig_shape = tuple((header['height'], header['width']))
elif int(np.prod(
self._sig_shape)) != (header['height'] * header['width']):
raise DataSetException("sig_shape must be of size: %s" %
(header['height'] * header['width']))
self._nav_shape_product = int(np.prod(self._nav_shape))
self._sync_offset_info = self.get_sync_offset_info()
shape = Shape(self._nav_shape + self._sig_shape,
sig_dims=len(self._sig_shape))
self._meta = DataSetMeta(
shape=shape,
raw_dtype=dtype,
dtype=dtype,
metadata=header,
sync_offset=self._sync_offset,
image_count=self._image_count,
)
self._maybe_load_dark_gain()
return self
