def _do_map(self, path, offset, shape, dtype): length = np.product(shape) if self.nommap: if len(shape) > 1: raise RuntimeError('not supported, high d arrays from non local files') file = self._get_file(path) column = ColumnFile(file, offset, length, dtype, write=self.write, path=self.path, tls=self.tls_map[path]) else: mapping = self._get_mapping(path) column = np.frombuffer(mapping, dtype=dtype, count=length, offset=offset) column = column.reshape(shape) return column
def _map_array(self, offset=None, length=None, dtype=np.float64, stride=1, filename=None, array=None, name='unknown'): if filename is None: filename = self.filename if not self.nommap: mapping = self.mapping_map[filename] if array is not None: length = len(array) # if self._length_original is not None and length != self._length_original: # logger.error("inconsistent length", "length of column %s is %d, while %d was expected" % (name, length, self._length)) # else: # self._length_unfiltered = length # self._length_original = length # if self.current_slice is None: # self.current_slice = (0, length) # self.fraction = 1. # self._length = length # self._index_end = self._length_unfiltered # self._index_start = 0 # print self.mapping, dtype, length if stride is None else length * stride, offset if 1: if array is not None: length = len(array) mmapped_array = array stride = None offset = None dtype = array.dtype column = array else: if offset is None: print("offset is None") sys.exit(0) file = self.file_map[filename] if self.nommap: column = ColumnFile(file, offset, length, dtype, write=self.write, path=self.path) else: column = np.frombuffer( mapping, dtype=dtype, count=length if stride is None else length * stride, offset=offset) if stride and stride != 1: column = column[::stride] return column
def mmap_array(mmap, file, offset, dtype, shape): length = np.product(shape) if mmap is None: if len(shape) > 1: raise RuntimeError( 'not supported, high d arrays from non local files') return ColumnFile(file, offset, length, dtype, write=True, tls=None) else: array = np.frombuffer(mmap, dtype=dtype, count=length, offset=offset) return array.reshape(shape)
def _do_map(self, path, offset, length, dtype): if self.nommap: file = self._get_file(path) column = ColumnFile(file, offset, length, dtype, write=self.write, path=self.path) else: mapping = self._get_mapping(path) column = np.frombuffer(mapping, dtype=dtype, count=length, offset=offset) return column
def mmap_array(mmap, file, offset, dtype, length): if mmap is None: return ColumnFile(file, offset, length, dtype, write=True, tls=None) else: return np.frombuffer(mmap, dtype=dtype, count=length, offset=offset)