def totif(images, path, prefix="image", overwrite=False, credentials=None): """ Write out TIF files for 2d image data. See also -------- thunder.data.images.totif """ dims = images.dims if not len(dims) in [2, 3]: raise ValueError("Only 2D or 3D images can be exported to tif, " "images are %d-dimensional." % len(dims)) from scipy.misc import imsave from io import BytesIO from thunder.writers import get_parallel_writer def tobuffer(kv): key, img = kv fname = prefix + "-" + "%05d.tif" % int(key) bytebuf = BytesIO() imsave(bytebuf, img, format='TIFF') return fname, bytebuf.getvalue() writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) images.foreach(lambda x: writer.write(tobuffer(x)))
def totif(images, path, prefix="image", overwrite=False, credentials=None): """ Write out TIF files for 2d image data. See also -------- thunder.data.images.totif """ dims = images.dims if not len(dims) in [2, 3]: raise ValueError("Only 2D or 3D images can be exported to tif, " "images are %d-dimensional." % len(dims)) from scipy.misc import imsave from io import BytesIO from thunder.writers import get_parallel_writer def tobuffer(kv): key, img = kv fname = prefix+"-"+"%05d.tif" % int(key) bytebuf = BytesIO() imsave(bytebuf, img, format='TIFF') return fname, bytebuf.getvalue() writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) images.foreach(lambda x: writer.write(tobuffer(x)))
def tobinary(images, path, prefix="image", overwrite=False, credentials=None): """ Write out images as binary files. See also -------- thunder.data.images.tobinary """ from thunder.writers import get_parallel_writer def tobuffer(kv): key, img = kv fname = prefix + "-" + "%05d.bin" % int(key) return fname, img.copy() writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) images.foreach(lambda x: writer.write(tobuffer(x))) config(path, list(images.dims), images.dtype, overwrite=overwrite)
def tobinary(images, path, prefix="image", overwrite=False, credentials=None): """ Write out images as binary files. See also -------- thunder.data.images.tobinary """ from thunder.writers import get_parallel_writer def tobuffer(kv): key, img = kv fname = prefix + "-" + "%05d.bin" % int(key) return fname, img.copy() writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) images.foreach(lambda x: writer.write(tobuffer(x))) config(path, list(images.dims), images.dtype, overwrite=overwrite)
def tobinary(series, path, prefix='series', overwrite=False, credentials=None): """ Writes out data to binary format. Parameters ---------- series : Series The data to write path : string path or URI to directory to be created Output files will be written underneath path. Directory will be created as a result of this call. prefix : str, optional, default = 'series' String prefix for files. overwrite : bool If true, path and all its contents will be deleted and recreated as partof this call. """ from six import BytesIO from thunder.utils import check_path from thunder.writers import get_parallel_writer if not overwrite: check_path(path, credentials=credentials) overwrite = True def tobuffer(kv): firstkey = None buf = BytesIO() for k, v in kv: if firstkey is None: firstkey = k buf.write(v.tostring()) val = buf.getvalue() buf.close() if firstkey is None: return iter([]) else: label = prefix + '-' + getlabel(firstkey) + ".bin" return iter([(label, val)]) writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) if series.mode == 'spark': binary = series.values.tordd().sortByKey().mapPartitions(tobuffer) binary.foreach(writer.write) else: basedims = [series.shape[d] for d in series.baseaxes] def split(k): ind = unravel_index(k, basedims) return ind, series.values[ind] buf = tobuffer([split(i) for i in range(prod(basedims))]) [writer.write(b) for b in buf] shape = series.shape dtype = series.dtype write_config(path, shape=shape, dtype=dtype, overwrite=overwrite, credentials=credentials)
def tobinary(series, path, prefix='series', overwrite=False, credentials=None): """ Writes out data to binary format. Parameters ---------- series : Series The data to write path : string path or URI to directory to be created Output files will be written underneath path. Directory will be created as a result of this call. prefix : str, optional, default = 'series' String prefix for files. overwrite : bool If true, path and all its contents will be deleted and recreated as partof this call. """ from six import BytesIO from thunder.utils import check_path from thunder.writers import get_parallel_writer if not overwrite: check_path(path, credentials=credentials) overwrite = True def tobuffer(kv): firstkey = None buf = BytesIO() for k, v in kv: if firstkey is None: firstkey = k buf.write(v.tostring()) val = buf.getvalue() buf.close() if firstkey is None: return iter([]) else: label = prefix + '-' + getlabel(firstkey) + ".bin" return iter([(label, val)]) writer = get_parallel_writer(path)(path, overwrite=overwrite, credentials=credentials) if series.mode == 'spark': binary = series.values.tordd().sortByKey().mapPartitions(tobuffer) binary.foreach(writer.write) else: basedims = [series.shape[d] for d in series.baseaxes] def split(k): ind = unravel_index(k, basedims) return ind, series.values[ind] buf = tobuffer([split(i) for i in range(prod(basedims))]) [writer.write(b) for b in buf] shape = series.shape dtype = series.dtype write_config(path, shape=shape, dtype=dtype, overwrite=overwrite, credentials=credentials)