def fromTif(self, datapath, ext='tif', startidx=None, stopidx=None):
"""Load an Images object stored in a directory of (single-page) tif files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Parameters
----------
datapath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
ext: string, optional, default "tif"
Extension required on data files to be loaded.
startidx, stopidx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
"""
def readTifFromBuf(buf):
fbuf = BytesIO(buf)
return imread(fbuf, format='tif')
reader = getParallelReaderForPath(datapath)(self.sc)
readerrdd = reader.read(datapath, ext=ext, startidx=startidx, stopidx=stopidx)
return Images(readerrdd.mapValues(readTifFromBuf), nimages=reader.lastnrecs)
def fromPng(self, dataPath, ext='png', startIdx=None, stopIdx=None, recursive=False):
"""Load an Images object stored in a directory of png files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
ext: string, optional, default "png"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
"""
def readPngFromBuf(buf):
fbuf = BytesIO(buf)
return imread(fbuf, format='png')
reader = getParallelReaderForPath(dataPath)(self.sc)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive)
return Images(readerRdd.mapValues(readPngFromBuf), nimages=reader.lastNRecs)
def fromStack(self, dataPath, dims, dtype='int16', ext='stack', startIdx=None, stopIdx=None, recursive=False):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
"""
if not dims:
raise ValueError("Image dimensions must be specified if loading from binary stack data")
def toArray(buf):
return frombuffer(buf, dtype=dtype, count=int(prod(dims))).reshape(dims, order='F')
reader = getParallelReaderForPath(dataPath)(self.sc)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive)
return Images(readerRdd.mapValues(toArray), nimages=reader.lastNRecs, dims=dims,
dtype=dtype)
def fromPng(self, dataPath, ext='png', startIdx=None, stopIdx=None, recursive=False, npartitions=None):
"""Load an Images object stored in a directory of png files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A dataPath argument may include a single '*' wildcard character in the filename.
ext: string, optional, default "png"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
npartitions: positive int, optional.
If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
partition per image file.
"""
def readPngFromBuf(buf):
fbuf = BytesIO(buf)
return imread(fbuf, format='png')
reader = getParallelReaderForPath(dataPath)(self.sc, awsCredentialsOverride=self.awsCredentialsOverride)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive,
npartitions=npartitions)
return Images(readerRdd.mapValues(readPngFromBuf), nrecords=reader.lastNRecs)
def fromStack(self, datapath, dims, dtype='int16', ext='stack', startidx=None, stopidx=None):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
datapath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startidx, stopidx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
"""
if not dims:
raise ValueError("Image dimensions must be specified if loading from binary stack data")
def toArray(buf):
return frombuffer(buf, dtype=dtype, count=int(prod(dims))).reshape(dims, order='F')
reader = getParallelReaderForPath(datapath)(self.sc)
readerrdd = reader.read(datapath, ext=ext, startidx=startidx, stopidx=stopidx)
return Images(readerrdd.mapValues(toArray), nimages=reader.lastnrecs, dims=dims,
dtype=dtype)
def fromStack(self,
dataPath,
dims=None,
dtype=None,
ext='stack',
startIdx=None,
stopIdx=None,
recursive=False,
nplanes=None,
npartitions=None,
confFilename='conf.json'):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A dataPath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
nplanes: positive integer, default None
If passed, will cause a single binary stack file to be subdivided into multiple records. Every
`nplanes` z-planes in the file will be taken as a new record, with the first nplane planes of the
first file being record 0, the second nplane planes being record 1, etc, until the first file is
exhausted and record ordering continues with the first nplane planes of the second file, and so on.
With nplanes=None (the default), a single file will be considered as representing a single record.
npartitions: positive int, optional.
If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
partition per image file.
"""
reader = getFileReaderForPath(dataPath)(
awsCredentialsOverride=self.awsCredentialsOverride)
try:
jsonBuf = reader.read(dataPath, filename=confFilename)
params = json.loads(jsonBuf)
except FileNotFoundError:
params = {}
if 'dtype' in params.keys():
dtype = params['dtype']
if 'dims' in params.keys():
dims = params['dims']
if not dims:
raise ValueError(
"Image dimensions must be specified either as argument or in a conf.json file"
)
if not dtype:
dtype = 'int16'
if nplanes is not None:
if nplanes <= 0:
raise ValueError("nplanes must be positive if passed, got %d" %
nplanes)
if dims[-1] % nplanes:
raise ValueError(
"Last dimension of stack image '%d' must be divisible by nplanes '%d'"
% (dims[-1], nplanes))
def toArray(idxAndBuf):
idx, buf = idxAndBuf
ary = frombuffer(buf, dtype=dtype,
count=int(prod(dims))).reshape(dims, order='F')
if nplanes is None:
yield idx, ary
else:
# divide array into chunks of nplanes
npoints = dims[-1] / nplanes # integer division
if dims[-1] % nplanes:
npoints += 1
timepoint = 0
lastPlane = 0
curPlane = 1
while curPlane < ary.shape[-1]:
if curPlane % nplanes == 0:
slices = [slice(None)] * (ary.ndim - 1) + [
slice(lastPlane, curPlane)
]
yield idx * npoints + timepoint, ary[slices]
timepoint += 1
lastPlane = curPlane
curPlane += 1
# yield remaining planes
slices = [slice(None)] * (ary.ndim - 1) + [
slice(lastPlane, ary.shape[-1])
]
yield idx * npoints + timepoint, ary[slices]
reader = getParallelReaderForPath(dataPath)(
self.sc, awsCredentialsOverride=self.awsCredentialsOverride)
readerRdd = reader.read(dataPath,
ext=ext,
startIdx=startIdx,
stopIdx=stopIdx,
recursive=recursive,
npartitions=npartitions)
nrecords = reader.lastNRecs if nplanes is None else None
newDims = tuple(list(dims[:-1]) + [nplanes]) if nplanes else dims
return Images(readerRdd.flatMap(toArray),
nrecords=nrecords,
dims=newDims,
dtype=dtype)
class ImagesLoader(object):
"""Loader object used to instantiate Images data stored in a variety of formats.
"""
def __init__(self, sparkContext):
"""Initialize a new ImagesLoader object.
Parameters
----------
sparkcontext: SparkContext
The pyspark SparkContext object used by the current Thunder environment.
"""
from thunder.utils.common import AWSCredentials
self.sc = sparkContext
self.awsCredentialsOverride = AWSCredentials.fromContext(sparkContext)
def fromArrays(self, arrays, npartitions=None):
"""Load Images data from passed sequence of numpy arrays.
Expected usage is mainly in testing - having a full dataset loaded in memory
on the driver is likely prohibitive in the use cases for which Thunder is intended.
"""
# if passed a single array, cast it to a sequence of length 1
if isinstance(arrays, ndarray):
arrays = [arrays]
shape = None
dtype = None
for ary in arrays:
if shape is None:
shape = ary.shape
dtype = ary.dtype
if not ary.shape == shape:
raise ValueError(
"Arrays must all be of same shape; got both %s and %s" %
(str(shape), str(ary.shape)))
if not ary.dtype == dtype:
raise ValueError(
"Arrays must all be of same data type; got both %s and %s"
% (str(dtype), str(ary.dtype)))
narrays = len(arrays)
npartitions = min(narrays, npartitions) if npartitions else narrays
return Images(self.sc.parallelize(enumerate(arrays), npartitions),
dims=shape,
dtype=str(dtype),
nrecords=narrays)
def fromStack(self,
dataPath,
dims=None,
dtype=None,
ext='stack',
startIdx=None,
stopIdx=None,
recursive=False,
nplanes=None,
npartitions=None,
confFilename='conf.json'):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A dataPath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
nplanes: positive integer, default None
If passed, will cause a single binary stack file to be subdivided into multiple records. Every
`nplanes` z-planes in the file will be taken as a new record, with the first nplane planes of the
first file being record 0, the second nplane planes being record 1, etc, until the first file is
exhausted and record ordering continues with the first nplane planes of the second file, and so on.
With nplanes=None (the default), a single file will be considered as representing a single record.
npartitions: positive int, optional.
If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
partition per image file.
"""
reader = getFileReaderForPath(dataPath)(
awsCredentialsOverride=self.awsCredentialsOverride)
try:
jsonBuf = reader.read(dataPath, filename=confFilename)
params = json.loads(jsonBuf)
except FileNotFoundError:
params = {}
if 'dtype' in params.keys():
dtype = params['dtype']
if 'dims' in params.keys():
dims = params['dims']
if not dims:
raise ValueError(
"Image dimensions must be specified either as argument or in a conf.json file"
)
if not dtype:
dtype = 'int16'
if nplanes is not None:
if nplanes <= 0:
raise ValueError("nplanes must be positive if passed, got %d" %
nplanes)
if dims[-1] % nplanes:
raise ValueError(
"Last dimension of stack image '%d' must be divisible by nplanes '%d'"
% (dims[-1], nplanes))
def toArray(idxAndBuf):
idx, buf = idxAndBuf
ary = frombuffer(buf, dtype=dtype,
count=int(prod(dims))).reshape(dims, order='F')
if nplanes is None:
yield idx, ary
else:
# divide array into chunks of nplanes
npoints = dims[-1] / nplanes # integer division
if dims[-1] % nplanes:
npoints += 1
timepoint = 0
lastPlane = 0
curPlane = 1
while curPlane < ary.shape[-1]:
if curPlane % nplanes == 0:
slices = [slice(None)] * (ary.ndim - 1) + [
slice(lastPlane, curPlane)
]
yield idx * npoints + timepoint, ary[slices]
timepoint += 1
lastPlane = curPlane
curPlane += 1
# yield remaining planes
slices = [slice(None)] * (ary.ndim - 1) + [
slice(lastPlane, ary.shape[-1])
]
yield idx * npoints + timepoint, ary[slices]
reader = getParallelReaderForPath(dataPath)(
self.sc, awsCredentialsOverride=self.awsCredentialsOverride)
readerRdd = reader.read(dataPath,
ext=ext,
startIdx=startIdx,
stopIdx=stopIdx,
recursive=recursive,
npartitions=npartitions)
nrecords = reader.lastNRecs if nplanes is None else None
newDims = tuple(list(dims[:-1]) + [nplanes]) if nplanes else dims
return Images(readerRdd.flatMap(toArray),
nrecords=nrecords,
dims=newDims,
dtype=dtype)
def fromTif(self,
dataPath,
ext='tif',
startIdx=None,
stopIdx=None,
recursive=False,
nplanes=None,
npartitions=None):
"""Sets up a new Images object with data to be read from one or more tif files.
Multiple pages of a multipage tif file will by default be assumed to represent the z-axis (depth) of a
single 3-dimensional volume, in which case a single input multipage tif file will be converted into
a single Images record. If `nplanes` is passed, then every nplanes pages will be interpreted as a single
3d volume (2d if nplanes==1), allowing a single tif file to contain multiple Images records.
This method attempts to explicitly import PIL. ImportError may be thrown if 'from PIL import Image' is
unsuccessful. (PIL/pillow is not an explicit requirement for thunder.)
The RDD wrapped by the returned Images object will by default have a number of partitions equal to the
number of image data files read in by this method; it may have fewer partitions if npartitions is specified.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
ext: string, optional, default "tif"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
nplanes: positive integer, default None
If passed, will cause a single multipage tif file to be subdivided into multiple records. Every
`nplanes` tif pages in the file will be taken as a new record, with the first nplane pages of the
first file being record 0, the second nplane pages being record 1, etc, until the first file is
exhausted and record ordering continues with the first nplane images of the second file, and so on.
With nplanes=None (the default), a single file will be considered as representing a single record.
npartitions: positive int, optional.
If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
partition per image file.
"""
try:
from PIL import Image
except ImportError, e:
Image = None
raise ImportError(
"fromMultipageTif requires a successful 'from PIL import Image'; "
+ "the PIL/pillow library appears to be missing or broken.", e)
# we know that that array(pilimg) works correctly for pillow == 2.3.0, and that it
# does not work (at least not with spark) for old PIL == 1.1.7. we believe but have not confirmed
# that array(pilimg) works correctly for every version of pillow. thus currently we check only whether
# our PIL library is in fact pillow, and choose our conversion function accordingly
isPillow = hasattr(Image, "PILLOW_VERSION")
if isPillow:
conversionFcn = array # use numpy's array() function
else:
from thunder.utils.common import pil_to_array
conversionFcn = pil_to_array # use our modified version of matplotlib's pil_to_array
if nplanes is not None and nplanes <= 0:
raise ValueError("nplanes must be positive if passed, got %d" %
nplanes)
def multitifReader(idxAndBuf):
idx, buf = idxAndBuf
pageCount = -1
values = []
fbuf = BytesIO(buf)
multipage = Image.open(fbuf)
if multipage.mode.startswith('I') and 'S' in multipage.mode:
# signed integer tiff file; use tifffile module to read
import thunder.rdds.fileio.tifffile as tifffile
fbuf.seek(0) # reset pointer after read done by PIL
tfh = tifffile.TiffFile(fbuf)
ary = tfh.asarray(
) # ary comes back with pages as first dimension, will need to transpose
pageCount = ary.shape[0]
if nplanes is not None:
values = [
ary[i:(i + nplanes)]
for i in xrange(0, ary.shape[0], nplanes)
]
else:
values = [ary]
tfh.close()
# transpose Z dimension if any, leave X and Y in same order
if ary.ndim == 3:
values = [val.transpose((1, 2, 0)) for val in values]
# squeeze out last dimension if singleton
values = [
val.squeeze(-1) if val.shape[-1] == 1 else val
for val in values
]
else:
# normal case; use PIL/Pillow for anything but signed ints
pageIdx = 0
imgArys = []
npagesLeft = -1 if nplanes is None else nplanes # counts number of planes remaining in image if positive
while True:
try:
multipage.seek(pageIdx)
imgArys.append(conversionFcn(multipage))
pageIdx += 1
npagesLeft -= 1
if npagesLeft == 0:
# we have just finished an image from this file
retAry = dstack(
imgArys) if len(imgArys) > 1 else imgArys[0]
values.append(retAry)
# reset counters:
npagesLeft = nplanes
imgArys = []
except EOFError:
# past last page in tif
break
pageCount = pageIdx
if imgArys:
retAry = dstack(
imgArys) if len(imgArys) > 1 else imgArys[0]
values.append(retAry)
# check for inappropriate nplanes that doesn't evenly divide num pages
if nplanes and (pageCount % nplanes):
raise ValueError(
"nplanes '%d' does not evenly divide page count of multipage tif '%d'"
% (nplanes, pageCount))
nvals = len(values)
keys = [idx * nvals + timepoint for timepoint in xrange(nvals)]
return zip(keys, values)
reader = getParallelReaderForPath(dataPath)(
self.sc, awsCredentialsOverride=self.awsCredentialsOverride)
readerRdd = reader.read(dataPath,
ext=ext,
startIdx=startIdx,
stopIdx=stopIdx,
recursive=recursive,
npartitions=npartitions)
nrecords = reader.lastNRecs if nplanes is None else None
return Images(readerRdd.flatMap(multitifReader), nrecords=nrecords)
def fromStack(self, dataPath, dims=None, dtype=None, ext='stack', startIdx=None, stopIdx=None, recursive=False,
nplanes=None, npartitions=None, confFilename='conf.json'):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A dataPath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
nplanes: positive integer, default None
If passed, will cause a single binary stack file to be subdivided into multiple records. Every
`nplanes` z-planes in the file will be taken as a new record, with the first nplane planes of the
first file being record 0, the second nplane planes being record 1, etc, until the first file is
exhausted and record ordering continues with the first nplane planes of the second file, and so on.
With nplanes=None (the default), a single file will be considered as representing a single record.
npartitions: positive int, optional.
If specified, request a certain number of partitions for the underlying Spark RDD. Default is 1
partition per image file.
"""
reader = getFileReaderForPath(dataPath)(awsCredentialsOverride=self.awsCredentialsOverride)
try:
jsonBuf = reader.read(dataPath, filename=confFilename)
params = json.loads(jsonBuf)
except FileNotFoundError:
params = {}
if 'dtype' in params.keys():
dtype = params['dtype']
if 'dims' in params.keys():
dims = params['dims']
if not dims:
raise ValueError("Image dimensions must be specified either as argument or in a conf.json file")
if not dtype:
dtype = 'int16'
if nplanes is not None:
if nplanes <= 0:
raise ValueError("nplanes must be positive if passed, got %d" % nplanes)
if dims[-1] % nplanes:
raise ValueError("Last dimension of stack image '%d' must be divisible by nplanes '%d'" %
(dims[-1], nplanes))
def toArray(idxAndBuf):
idx, buf = idxAndBuf
ary = frombuffer(buf, dtype=dtype, count=int(prod(dims))).reshape(dims, order='F')
if nplanes is None:
yield idx, ary
else:
# divide array into chunks of nplanes
npoints = dims[-1] / nplanes # integer division
if dims[-1] % nplanes:
npoints += 1
timepoint = 0
lastPlane = 0
curPlane = 1
while curPlane < ary.shape[-1]:
if curPlane % nplanes == 0:
slices = [slice(None)] * (ary.ndim - 1) + [slice(lastPlane, curPlane)]
yield idx*npoints + timepoint, ary[slices]
timepoint += 1
lastPlane = curPlane
curPlane += 1
# yield remaining planes
slices = [slice(None)] * (ary.ndim - 1) + [slice(lastPlane, ary.shape[-1])]
yield idx*npoints + timepoint, ary[slices]
reader = getParallelReaderForPath(dataPath)(self.sc, awsCredentialsOverride=self.awsCredentialsOverride)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive,
npartitions=npartitions)
nrecords = reader.lastNRecs if nplanes is None else None
newDims = tuple(list(dims[:-1]) + [nplanes]) if nplanes else dims
return Images(readerRdd.flatMap(toArray), nrecords=nrecords, dims=newDims, dtype=dtype)
class ImagesLoader(object):
"""Loader object used to instantiate Images data stored in a variety of formats.
"""
def __init__(self, sparkContext):
"""Initialize a new ImagesLoader object.
Parameters
----------
sparkcontext: SparkContext
The pyspark SparkContext object used by the current Thunder environment.
"""
self.sc = sparkContext
def fromArrays(self, arrays):
"""Load Images data from passed sequence of numpy arrays.
Expected usage is mainly in testing - having a full dataset loaded in memory
on the driver is likely prohibitive in the use cases for which Thunder is intended.
"""
# if passed a single array, cast it to a sequence of length 1
if isinstance(arrays, ndarray):
arrays = [arrays]
shape = None
dtype = None
for ary in arrays:
if shape is None:
shape = ary.shape
dtype = ary.dtype
if not ary.shape == shape:
raise ValueError("Arrays must all be of same shape; got both %s and %s" %
(str(shape), str(ary.shape)))
if not ary.dtype == dtype:
raise ValueError("Arrays must all be of same data type; got both %s and %s" %
(str(dtype), str(ary.dtype)))
return Images(self.sc.parallelize(enumerate(arrays), len(arrays)),
dims=shape, dtype=str(dtype), nimages=len(arrays))
def fromStack(self, dataPath, dims, dtype='int16', ext='stack', startIdx=None, stopIdx=None, recursive=False):
"""Load an Images object stored in a directory of flat binary files
The RDD wrapped by the returned Images object will have a number of partitions equal to the number of image data
files read in by this method.
Currently all binary data read by this method is assumed to be formatted as signed 16 bit integers in native
byte order.
Parameters
----------
dataPath: string
Path to data files or directory, specified as either a local filesystem path or in a URI-like format,
including scheme. A datapath argument may include a single '*' wildcard character in the filename.
dims: tuple of positive int
Dimensions of input image data, ordered with fastest-changing dimension first
ext: string, optional, default "stack"
Extension required on data files to be loaded.
startIdx, stopIdx: nonnegative int. optional.
Indices of the first and last-plus-one data file to load, relative to the sorted filenames matching
`datapath` and `ext`. Interpreted according to python slice indexing conventions.
recursive: boolean, default False
If true, will recursively descend directories rooted at datapath, loading all files in the tree that
have an extension matching 'ext'. Recursive loading is currently only implemented for local filesystems
(not s3).
"""
if not dims:
raise ValueError("Image dimensions must be specified if loading from binary stack data")
def toArray(buf):
return frombuffer(buf, dtype=dtype, count=int(prod(dims))).reshape(dims, order='F')
reader = getParallelReaderForPath(dataPath)(self.sc)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive)
return Images(readerRdd.mapValues(toArray), nimages=reader.lastNRecs, dims=dims,
dtype=dtype)
def fromTif(self, dataPath, ext='tif', startIdx=None, stopIdx=None, recursive=False):
"""Sets up a new Images object with data to be read from one or more tif files.
The RDD underlying the returned Images will have key, value data as follows:
key: int
key is index of original data file, determined by lexicographic ordering of filenames
value: numpy ndarray
value dimensions with be x by y by num_channels*num_pages; all channels and pages in a file are
concatenated together in the third dimension of the resulting ndarray. For pages 0, 1, etc
of a multipage TIF of RGB images, ary[:,:,0] will be R channel of page 0 ("R0"), ary[:,:,1] will be B0,
... ary[:,:,3] == R1, and so on.
This method attempts to explicitly import PIL. ImportError may be thrown if 'from PIL import Image' is
unsuccessful. (PIL/pillow is not an explicit requirement for thunder.)
"""
try:
from PIL import Image
except ImportError, e:
Image = None
raise ImportError("fromMultipageTif requires a successful 'from PIL import Image'; " +
"the PIL/pillow library appears to be missing or broken.", e)
# we know that that array(pilimg) works correctly for pillow == 2.3.0, and that it
# does not work (at least not with spark) for old PIL == 1.1.7. we believe but have not confirmed
# that array(pilimg) works correctly for every version of pillow. thus currently we check only whether
# our PIL library is in fact pillow, and choose our conversion function accordingly
isPillow = hasattr(Image, "PILLOW_VERSION")
if isPillow:
conversionFcn = array # use numpy's array() function
else:
from thunder.utils.common import pil_to_array
conversionFcn = pil_to_array # use our modified version of matplotlib's pil_to_array
def multitifReader(buf):
fbuf = BytesIO(buf)
multipage = Image.open(fbuf)
pageIdx = 0
imgArys = []
while True:
try:
multipage.seek(pageIdx)
imgArys.append(conversionFcn(multipage))
pageIdx += 1
except EOFError:
# past last page in tif
break
if len(imgArys) == 1:
return imgArys[0]
else:
return dstack(imgArys)
reader = getParallelReaderForPath(dataPath)(self.sc)
readerRdd = reader.read(dataPath, ext=ext, startIdx=startIdx, stopIdx=stopIdx, recursive=recursive)
return Images(readerRdd.mapValues(multitifReader), nimages=reader.lastNRecs)