def _do_map(self, path, offset, shape, dtype):
length = np.product(shape)
if self.nommap:
if len(shape) > 1:
raise RuntimeError('not supported, high d arrays from non local files')
file = self._get_file(path)
column = ColumnFile(file, offset, length, dtype, write=self.write, path=self.path, tls=self.tls_map[path])
else:
mapping = self._get_mapping(path)
column = np.frombuffer(mapping, dtype=dtype, count=length, offset=offset)
column = column.reshape(shape)
return column
def _map_array(self,
offset=None,
length=None,
dtype=np.float64,
stride=1,
filename=None,
array=None,
name='unknown'):
if filename is None:
filename = self.filename
if not self.nommap:
mapping = self.mapping_map[filename]
if array is not None:
length = len(array)
# if self._length_original is not None and length != self._length_original:
# logger.error("inconsistent length", "length of column %s is %d, while %d was expected" % (name, length, self._length))
# else:
# self._length_unfiltered = length
# self._length_original = length
# if self.current_slice is None:
# self.current_slice = (0, length)
# self.fraction = 1.
# self._length = length
# self._index_end = self._length_unfiltered
# self._index_start = 0
# print self.mapping, dtype, length if stride is None else length * stride, offset
if 1:
if array is not None:
length = len(array)
mmapped_array = array
stride = None
offset = None
dtype = array.dtype
column = array
else:
if offset is None:
print("offset is None")
sys.exit(0)
file = self.file_map[filename]
if self.nommap:
column = ColumnFile(file,
offset,
length,
dtype,
write=self.write,
path=self.path)
else:
column = np.frombuffer(
mapping,
dtype=dtype,
count=length if stride is None else length * stride,
offset=offset)
if stride and stride != 1:
column = column[::stride]
return column
def mmap_array(mmap, file, offset, dtype, shape):
length = np.product(shape)
if mmap is None:
if len(shape) > 1:
raise RuntimeError(
'not supported, high d arrays from non local files')
return ColumnFile(file, offset, length, dtype, write=True, tls=None)
else:
array = np.frombuffer(mmap, dtype=dtype, count=length, offset=offset)
return array.reshape(shape)
def _do_map(self, path, offset, length, dtype):
if self.nommap:
file = self._get_file(path)
column = ColumnFile(file,
offset,
length,
dtype,
write=self.write,
path=self.path)
else:
mapping = self._get_mapping(path)
column = np.frombuffer(mapping,
dtype=dtype,
count=length,
offset=offset)
return column
def mmap_array(mmap, file, offset, dtype, length):
if mmap is None:
return ColumnFile(file, offset, length, dtype, write=True, tls=None)
else:
return np.frombuffer(mmap, dtype=dtype, count=length, offset=offset)