def __init__(self, dyndarr): if dyndarr.undim <= 0: raise IndexError('Need at least one dimension for iteration') self._index = 0 self._len = len(dyndarr) ds = datashape.dshape(dyndarr.dshape) self._dshape = ds.subarray(1) self._c_dtype = nd.dtype(str(self._dshape)) self._usebuffer = (nd.dtype(str(ds)) != dyndarr.dtype) self._buffer = None self._buffer_index = -1 self.dyndarr = dyndarr
def __init__(self, dyndarr, nindex): if nindex > dyndarr.undim: raise IndexError('Cannot have more indices than dimensions') self._nindex = nindex self._dshape = datashape.dshape(dyndarr.dshape).subarray(nindex) self._c_dtype = nd.dtype(str(self._dshape)) self.dyndarr = dyndarr
def __init__(self, url, dshape=None): self.url = url if dshape is None: self.dshape = requests.get_remote_datashape(url) else: self.dshape = dshape self.dtype = nd.dtype(self.dshape)
def __init__(self, dyndarr): if dyndarr.undim <= 0: raise IndexError('Need at least one dimension for iteration') self._index = 0 self._len = len(dyndarr) self._dshape = datashape.dshape(dyndarr.dshape).subarray(1) self._c_dtype = nd.dtype(str(self._dshape)) self.dyndarr = dyndarr
def test_strided_scalar_assign(self): # Make a CKernel that assigns fixed-size strings to float32 ck = CKernel(UnaryStridedOperation) _lowlevel.py_api.make_assignment_kernel( ndt.float32, nd.dtype('string(15,"A")'), 'strided', ctypes.addressof(ck.dynamic_kernel_instance)) # Do an assignment using a numpy array src = np.array(['3.25', '-1000', '1e5'], dtype='S15') dst = np.arange(3, dtype=np.float32) ck(dst.ctypes.data, 4, src.ctypes.data, 15, 3) self.assertEqual(dst.tolist(), [3.25, -1000, 1e5])
def load_json_file_array(root, array_name): # Load the datashape dsfile = root + '.datashape' if not path.isfile(dsfile): dsfile = path.dirname(root) + '.datashape' if not path.isfile(dsfile): raise Exception('No datashape file found for array %s' % array_name) with open(dsfile) as f: dt = nd.dtype(f.read()) # Load the JSON with open(root + '.json') as f: # TODO: Add stream support to parse_json for compressed JSON, etc. arr = nd.parse_json(dt, f.read()) return arr
def load_json_directory_array(root, array_name): # Load the datashape dsfile = root + '.datashape' if not path.isfile(dsfile): raise Exception('No datashape file found for array %s' % array_name) with open(dsfile) as f: dt = nd.dtype(f.read()) # Scan for JSON files, assuming they're just #.json # Sort them numerically files = sorted([(int(path.splitext(path.basename(x))[0]), x) for x in glob.glob(path.join(root, '*.json'))]) files = [x[1] for x in files] # Make an array with an extra fixed dimension, then # read a JSON file into each element of that array dt = ndt.make_fixed_dim_dtype(len(files), dt) arr = nd.empty(dt) for i, fname in enumerate(files): with open(fname) as f: nd.parse_json(arr[i], f.read()) arr.flag_as_immutable() return arr
def load_json_file_list_array(root, array_name): # Load the datashape dsfile = root + '.datashape' if not path.isfile(dsfile): raise Exception('No datashape file found for array %s' % array_name) with open(dsfile) as f: dt = nd.dtype(f.read()) # Scan for JSON files -- no assumption on file suffix #open list of files and load into python list files = root + '.files' with open(files) as f: l_files = [fs.strip() for fs in f] # Make an array with an extra fixed dimension, then # read a JSON file into each element of that array dt = ndt.make_fixed_dim_dtype(len(l_files), dt) arr = nd.empty(dt) for i, fname in enumerate(l_files): with open(fname) as f: nd.parse_json(arr[i], f.read()) arr.flag_as_immutable() return arr
def array(obj, dshape=None, caps={'efficient-write': True}, storage=None): """Create a Blaze array. Parameters ---------- obj : array_like Initial contents for the array. dshape : datashape The datashape for the resulting array. By default the datashape will be inferred from data. If an explicit dshape is provided, the input data will be coerced into the provided dshape. caps : capabilities dictionary A dictionary containing the desired capabilities of the array. storage : Storage instance A Storage object with the necessary info for storing the data. Returns ------- out : a concrete blaze array. Bugs ---- Right now the explicit dshape is ignored. This needs to be corrected. When the data cannot be coerced to an explicit dshape an exception should be raised. """ dshape = _normalize_dshape(dshape) storage = _storage_convert(storage) if isinstance(obj, IDataDescriptor): # TODO: Validate the 'caps', convert to another kind # of data descriptor if necessary # Note by Francesc: but if it is already an IDataDescriptor I wonder # if `caps` should be ignored. Hmm, probably not... # # Note by Oscar: Maybe we shouldn't accept a datadescriptor at # all at this level. If you've got a DataDescriptor you are # playing with internal datastructures anyways, go to the # Array constructor directly. If you want to transform to # another datadescriptor... convert it yourself (you are # playing with internal datastructures, remember? you should # be able to do it in your own. dd = obj elif storage is not None: dt = None if dshape is None else to_numpy_dtype(dshape) if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor(blz.barray(obj, dtype=dt, count=-1, rootdir=storage.path)) else: dd = BLZDataDescriptor( blz.barray(obj, dtype=dt, rootdir=storage.path)) elif 'efficient-write' in caps and caps['efficient-write'] is True: # In-Memory array if dshape is None: dd = DyNDDataDescriptor(nd.array(obj)) else: # Use the uniform/full dtype specification in dynd depending # on whether the datashape has a uniform dim dt = nd.dtype(str(dshape)) if dt.undim > 0: dd = DyNDDataDescriptor(nd.array(obj, dtype=dt)) else: dd = DyNDDataDescriptor(nd.array(obj, udtype=dt)) elif 'compress' in caps and caps['compress'] is True: dt = None if dshape is None else to_numpy_dtype(dshape) # BLZ provides compression if inspect.isgenerator(obj): # TODO: Generator logic can go inside barray dd = BLZDataDescriptor(blz.fromiter(obj, dtype=dt, count=-1)) else: dd = BLZDataDescriptor(blz.barray(obj, dtype=dt)) elif isinstance(obj, np.ndarray): dd = DyNDDataDescriptor(nd.array(obj)) elif isinstance(obj, nd.array): dd = DyNDDataDescriptor(obj) elif isinstance(obj, blz.barray): dd = BLZDataDescriptor(obj) else: raise TypeError(('Failed to construct blaze array from ' 'object of type %r') % type(obj)) return Array(dd)