def __init__(self, dyndarr):
if dyndarr.undim <= 0:
raise IndexError('Need at least one dimension for iteration')
self._index = 0
self._len = len(dyndarr)
ds = datashape.dshape(dyndarr.dshape)
self._dshape = ds.subarray(1)
self._c_dtype = nd.dtype(str(self._dshape))
self._usebuffer = (nd.dtype(str(ds)) != dyndarr.dtype)
self._buffer = None
self._buffer_index = -1
self.dyndarr = dyndarr
def __init__(self, dyndarr, nindex):
if nindex > dyndarr.undim:
raise IndexError('Cannot have more indices than dimensions')
self._nindex = nindex
self._dshape = datashape.dshape(dyndarr.dshape).subarray(nindex)
self._c_dtype = nd.dtype(str(self._dshape))
self.dyndarr = dyndarr
def __init__(self, url, dshape=None):
self.url = url
if dshape is None:
self.dshape = requests.get_remote_datashape(url)
else:
self.dshape = dshape
self.dtype = nd.dtype(self.dshape)
def __init__(self, dyndarr):
if dyndarr.undim <= 0:
raise IndexError('Need at least one dimension for iteration')
self._index = 0
self._len = len(dyndarr)
self._dshape = datashape.dshape(dyndarr.dshape).subarray(1)
self._c_dtype = nd.dtype(str(self._dshape))
self.dyndarr = dyndarr
def test_strided_scalar_assign(self):
# Make a CKernel that assigns fixed-size strings to float32
ck = CKernel(UnaryStridedOperation)
_lowlevel.py_api.make_assignment_kernel(
ndt.float32, nd.dtype('string(15,"A")'), 'strided',
ctypes.addressof(ck.dynamic_kernel_instance))
# Do an assignment using a numpy array
src = np.array(['3.25', '-1000', '1e5'], dtype='S15')
dst = np.arange(3, dtype=np.float32)
ck(dst.ctypes.data, 4, src.ctypes.data, 15, 3)
self.assertEqual(dst.tolist(), [3.25, -1000, 1e5])
def load_json_file_array(root, array_name):
# Load the datashape
dsfile = root + '.datashape'
if not path.isfile(dsfile):
dsfile = path.dirname(root) + '.datashape'
if not path.isfile(dsfile):
raise Exception('No datashape file found for array %s' % array_name)
with open(dsfile) as f:
dt = nd.dtype(f.read())
# Load the JSON
with open(root + '.json') as f:
# TODO: Add stream support to parse_json for compressed JSON, etc.
arr = nd.parse_json(dt, f.read())
return arr
def load_json_directory_array(root, array_name):
# Load the datashape
dsfile = root + '.datashape'
if not path.isfile(dsfile):
raise Exception('No datashape file found for array %s' % array_name)
with open(dsfile) as f:
dt = nd.dtype(f.read())
# Scan for JSON files, assuming they're just #.json
# Sort them numerically
files = sorted([(int(path.splitext(path.basename(x))[0]), x)
for x in glob.glob(path.join(root, '*.json'))])
files = [x[1] for x in files]
# Make an array with an extra fixed dimension, then
# read a JSON file into each element of that array
dt = ndt.make_fixed_dim_dtype(len(files), dt)
arr = nd.empty(dt)
for i, fname in enumerate(files):
with open(fname) as f:
nd.parse_json(arr[i], f.read())
arr.flag_as_immutable()
return arr
def load_json_file_list_array(root, array_name):
# Load the datashape
dsfile = root + '.datashape'
if not path.isfile(dsfile):
raise Exception('No datashape file found for array %s' % array_name)
with open(dsfile) as f:
dt = nd.dtype(f.read())
# Scan for JSON files -- no assumption on file suffix
#open list of files and load into python list
files = root + '.files'
with open(files) as f:
l_files = [fs.strip() for fs in f]
# Make an array with an extra fixed dimension, then
# read a JSON file into each element of that array
dt = ndt.make_fixed_dim_dtype(len(l_files), dt)
arr = nd.empty(dt)
for i, fname in enumerate(l_files):
with open(fname) as f:
nd.parse_json(arr[i], f.read())
arr.flag_as_immutable()
return arr
def array(obj, dshape=None, caps={'efficient-write': True},
storage=None):
"""Create a Blaze array.
Parameters
----------
obj : array_like
Initial contents for the array.
dshape : datashape
The datashape for the resulting array. By default the
datashape will be inferred from data. If an explicit dshape is
provided, the input data will be coerced into the provided
dshape.
caps : capabilities dictionary
A dictionary containing the desired capabilities of the array.
storage : Storage instance
A Storage object with the necessary info for storing the data.
Returns
-------
out : a concrete blaze array.
Bugs
----
Right now the explicit dshape is ignored. This needs to be
corrected. When the data cannot be coerced to an explicit dshape
an exception should be raised.
"""
dshape = _normalize_dshape(dshape)
storage = _storage_convert(storage)
if isinstance(obj, IDataDescriptor):
# TODO: Validate the 'caps', convert to another kind
# of data descriptor if necessary
# Note by Francesc: but if it is already an IDataDescriptor I wonder
# if `caps` should be ignored. Hmm, probably not...
#
# Note by Oscar: Maybe we shouldn't accept a datadescriptor at
# all at this level. If you've got a DataDescriptor you are
# playing with internal datastructures anyways, go to the
# Array constructor directly. If you want to transform to
# another datadescriptor... convert it yourself (you are
# playing with internal datastructures, remember? you should
# be able to do it in your own.
dd = obj
elif storage is not None:
dt = None if dshape is None else to_numpy_dtype(dshape)
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(blz.barray(obj, dtype=dt, count=-1,
rootdir=storage.path))
else:
dd = BLZDataDescriptor(
blz.barray(obj, dtype=dt, rootdir=storage.path))
elif 'efficient-write' in caps and caps['efficient-write'] is True:
# In-Memory array
if dshape is None:
dd = DyNDDataDescriptor(nd.array(obj))
else:
# Use the uniform/full dtype specification in dynd depending
# on whether the datashape has a uniform dim
dt = nd.dtype(str(dshape))
if dt.undim > 0:
dd = DyNDDataDescriptor(nd.array(obj, dtype=dt))
else:
dd = DyNDDataDescriptor(nd.array(obj, udtype=dt))
elif 'compress' in caps and caps['compress'] is True:
dt = None if dshape is None else to_numpy_dtype(dshape)
# BLZ provides compression
if inspect.isgenerator(obj):
# TODO: Generator logic can go inside barray
dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1))
else:
dd = BLZDataDescriptor(blz.barray(obj, dtype=dt))
elif isinstance(obj, np.ndarray):
dd = DyNDDataDescriptor(nd.array(obj))
elif isinstance(obj, nd.array):
dd = DyNDDataDescriptor(obj)
elif isinstance(obj, blz.barray):
dd = BLZDataDescriptor(obj)
else:
raise TypeError(('Failed to construct blaze array from '
'object of type %r') % type(obj))
return Array(dd)
