def test_store_config_map_c(self) -> None:
sc1 = StoreConfig(index_depth=3, columns_depth=3)
maps = {'a': StoreConfig(index_depth=2),
'b': StoreConfig(index_depth=3)}
sc1m = StoreConfigMap(maps)
sc2m = StoreConfigMap.from_initializer(sc1)
self.assertEqual(sc2m['a'].index_depth, 3)
sc3m = StoreConfigMap.from_initializer(sc1m)
self.assertEqual(sc3m['a'].index_depth, 2)
self.assertEqual(sc3m['b'].index_depth, 3)
sc4m = StoreConfigMap.from_initializer(maps)
self.assertEqual(sc4m['a'].index_depth, 2)
self.assertEqual(sc4m['b'].index_depth, 3)
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None
):
'''
Args:
config: StoreConfig for handling ``Frame`` construction and exporting from Store.
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(f'supplied label {label} is not a string.')
if isinstance(value, Frame):
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
self._series = series
self._store = store
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def test_store_read_many_single_thread_weak_cache(self) -> None:
f1, f2, f3 = get_test_framesA()
with temp_file('.zip') as fp:
st = StoreZipTSV(fp)
st.write((f.name, f) for f in (f1, f2, f3))
kwargs = dict(config_map=StoreConfigMap.from_initializer(
StoreConfig(index_depth=1)),
constructor=st._container_type_to_constructor(Frame),
container_type=Frame)
labels = tuple(st.labels(strip_ext=False))
self.assertEqual(labels, ('foo.txt', 'bar.txt', 'baz.txt'))
self.assertEqual(0, len(list(st._weak_cache)))
# Result is not held onto!
next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(0, len(list(st._weak_cache)))
# Result IS held onto!
frame = next(st._read_many_single_thread(('foo', ), **kwargs))
self.assertEqual(1, len(list(st._weak_cache)))
# Reference in our weak_cache _is_ `frame`
self.assertIs(frame, st._weak_cache['foo'])
del frame
# Reference is gone now!
self.assertEqual(0, len(list(st._weak_cache)))
def write(self,
items: tp.Iterable[tp.Tuple[tp.Optional[str], Frame]],
*,
config: StoreConfigMapInitializer = None,
# store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT,
) -> None:
config_map = StoreConfigMap.from_initializer(config)
# NOTE: register adapters for NP types:
# numpy types go in as blobs if they are not individualy converted tp python types
sqlite3.register_adapter(np.int64, int)
sqlite3.register_adapter(np.int32, int)
sqlite3.register_adapter(np.int16, int)
# common python types
sqlite3.register_adapter(Fraction, str)
sqlite3.register_adapter(complex, lambda x: f'{x.real}:{x.imag}')
# hierarchical columns might be stored as tuples
with sqlite3.connect(self._fp, detect_types=sqlite3.PARSE_DECLTYPES) as conn:
cursor = conn.cursor()
for label, frame in items:
c = config_map[label]
self._frame_to_table(frame=frame,
label=label,
cursor=cursor,
include_columns=c.include_columns,
include_index=c.include_index,
# store_filter=store_filter
)
conn.commit()
def write(self,
items: tp.Iterable[tp.Tuple[tp.Hashable, Frame]],
*,
config: StoreConfigMapInitializer = None) -> None:
config_map = StoreConfigMap.from_initializer(config)
multiprocess = (config_map.default.write_max_workers is not None
and config_map.default.write_max_workers > 1)
def gen() -> tp.Iterable[PayloadFrameToBytes]:
for label, frame in items:
yield PayloadFrameToBytes( # pylint: disable=no-value-for-parameter
name=label,
config=config_map[label].to_store_config_he(),
frame=frame,
exporter=self.__class__._EXPORTER,
)
if multiprocess:
def label_and_bytes() -> tp.Iterator[LabelAndBytes]:
with ProcessPoolExecutor(max_workers=config_map.default.
write_max_workers) as executor:
yield from executor.map(
self._payload_to_bytes,
gen(),
chunksize=config_map.default.write_chunksize)
else:
label_and_bytes = lambda: (self._payload_to_bytes(x)
for x in gen())
with zipfile.ZipFile(self._fp, 'w', zipfile.ZIP_DEFLATED) as zf:
for label, frame_bytes in label_and_bytes():
label_encoded = config_map.default.label_encode(label)
# this will write it without a container
zf.writestr(label_encoded + self._EXT_CONTAINED, frame_bytes)
def read_many(self,
labels: tp.Iterable[tp.Hashable],
*,
config: StoreConfigMapInitializer = None,
container_type: tp.Type[Frame] = Frame,
) -> tp.Iterator[Frame]:
import tables
config_map = StoreConfigMap.from_initializer(config)
with tables.open_file(self._fp, mode='r') as file:
for label in labels:
c = config_map[label]
label_encoded = config_map.default.label_encode(label)
index_depth = c.index_depth
index_constructors = c.index_constructors
columns_depth = c.columns_depth
columns_constructors = c.columns_constructors
consolidate_blocks = c.consolidate_blocks
if c.dtypes:
raise NotImplementedError('using config.dtypes on HDF5 not yet supported')
index_arrays = []
columns_labels = []
table = file.get_node(f'/{label_encoded}')
colnames = table.cols._v_colnames
def blocks() -> tp.Iterator[np.ndarray]:
for col_idx, colname in enumerate(colnames):
# can also do: table.read(field=colname)
array = table.col(colname)
if array.dtype.kind in DTYPE_STR_KINDS:
array = array.astype(str)
array.flags.writeable = False
if col_idx < index_depth:
index_arrays.append(array)
continue
# only store column labels for those yielded
columns_labels.append(colname)
yield array
if consolidate_blocks:
data = TypeBlocks.from_blocks(TypeBlocks.consolidate_blocks(blocks()))
else:
data = TypeBlocks.from_blocks(blocks())
# this will own_data in subsequent constructor call
yield container_type._from_data_index_arrays_column_labels(
data=data,
index_depth=index_depth,
index_arrays=index_arrays,
index_constructors=index_constructors,
columns_depth=columns_depth,
columns_labels=columns_labels,
columns_constructors=columns_constructors,
name=label,
)
def test_store_config_map_get_default_a(self) -> None:
maps = {
'a': StoreConfig(index_depth=2),
'b': StoreConfig(index_depth=3)
}
sc1m = StoreConfigMap.from_initializer(maps)
self.assertTrue(sc1m.default == StoreConfigMap._DEFAULT)
def test_store_config_map_init_a(self) -> None:
maps = {
'a': StoreConfig(index_depth=2),
'b': StoreConfig(index_depth=3, label_encoder=str)
}
with self.assertRaises(ErrorInitStoreConfig):
sc1m = StoreConfigMap.from_initializer(maps)
def _from_store(
cls,
store: Store,
config: StoreConfigMapInitializer = None,
) -> 'Batch':
config_map = StoreConfigMap.from_initializer(config)
items = ((label, store.read(label, config=config_map[label]))
for label in store.labels())
return cls(items, config=config)
def _from_store(
cls,
store: Store,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None, # not used
) -> 'Batch':
config_map = StoreConfigMap.from_initializer(config)
items = ((label, store.read(label, config=config_map[label]))
for label in store.labels())
return cls(items, config=config)
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
own_data: bool = False,
):
'''
Default Bus constructor.
{args}
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
if max_persist is not None:
# use an (ordered) dictionary to give use an ordered set, simply pointing to None for all keys
self._last_accessed: tp.Dict[str, None] = {}
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if isinstance(value, Frame):
if max_persist is not None:
self._last_accessed[label] = None
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
if own_data:
self._values_mutable = series.values
self._values_mutable.flags.writeable = True
else:
self._values_mutable = series.values.copy()
self._index = series._index
self._name = series._name
self._store = store
# Not handling cases of max_persist being greater than the length of the Series (might floor to length)
if max_persist is not None and max_persist < self._loaded.sum():
raise ErrorInitBus('max_persist cannot be less than the number of already loaded Frames')
self._max_persist = max_persist
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def labels(
self,
*,
config: StoreConfigMapInitializer = None,
strip_ext: bool = True,
) -> tp.Iterator[tp.Hashable]:
config_map = StoreConfigMap.from_initializer(config)
with zipfile.ZipFile(self._fp) as zf:
for name in zf.namelist():
if strip_ext:
name = name.replace(self._EXT_CONTAINED, '')
# always use default decoder
yield config_map.default.label_decode(name)
def labels(
self,
*,
config: StoreConfigMapInitializer = None,
strip_ext: bool = True,
) -> tp.Iterator[tp.Hashable]:
config_map = StoreConfigMap.from_initializer(config)
wb = self._load_workbook(self._fp)
labels = tuple(wb.sheetnames)
wb.close()
for label in labels:
yield config_map.default.label_decode(label)
def labels(
self,
*,
config: StoreConfigMapInitializer = None,
strip_ext: bool = True,
) -> tp.Iterator[tp.Hashable]:
config_map = StoreConfigMap.from_initializer(config)
with sqlite3.connect(self._fp) as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table';")
for row in cursor:
yield config_map.default.label_decode(row[0])
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
):
'''
Args:
config: StoreConfig for handling :obj:`Frame` construction and exporting from Store.
max_persist: When loading :obj:`Frame` from a :obj:`Store`, optionally define the maximum number of :obj:`Frame` to remain in the :obj:`Bus`, regardless of the size of the :obj:`Bus`. If more than ``max_persist`` number of :obj:`Frame` are loaded, least-recently loaded :obj:`Frame` will be replaced by ``FrameDeferred``. A ``max_persist`` of 1, for example, permits reading one :obj:`Frame` at a time without ever holding in memory more than 1 :obj:`Frame`.
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
if max_persist is not None:
self._last_accessed: tp.Dict[str, None] = {}
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(f'supplied label {label} is not a string.')
if isinstance(value, Frame):
if max_persist is not None:
self._last_accessed[label] = None
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
self._series = series
self._store = store
# max_persist might be less than the number of Frames already loaded
if max_persist is not None:
self._max_persist = max(max_persist, self._loaded.sum())
else:
self._max_persist = None
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def labels(self, *,
config: StoreConfigMapInitializer = None,
strip_ext: bool = True,
) -> tp.Iterator[tp.Hashable]:
'''
Iterator of labels.
'''
import tables
config_map = StoreConfigMap.from_initializer(config)
with tables.open_file(self._fp, mode='r') as file:
for node in file.iter_nodes(where='/',
classname=tables.Table.__name__):
# NOTE: this is not the complete path
yield config_map.default.label_decode(node.name)
def write(
self,
items: tp.Iterable[tp.Tuple[tp.Hashable, Frame]],
*,
config: StoreConfigMapInitializer = None,
# store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT,
) -> None:
config_map = StoreConfigMap.from_initializer(config)
# NOTE: register adapters for NP types:
# numpy types go in as blobs if they are not individually converted tp python types
sqlite3.register_adapter(np.int64, int)
sqlite3.register_adapter(np.int32, int)
sqlite3.register_adapter(np.int16, int)
sqlite3.register_adapter(np.bool_, bool)
# common python types
sqlite3.register_adapter(Fraction, str)
sqlite3.register_adapter(complex, lambda x: f'{x.real}:{x.imag}')
# SQLite will naturally try to update, no replace, a DB found at an FP; this is not how all other stores work, so best to remove the file first.
with suppress(FileNotFoundError):
os.remove(self._fp)
# hierarchical columns might be stored as tuples
with sqlite3.connect(self._fp,
detect_types=sqlite3.PARSE_DECLTYPES) as conn:
cursor = conn.cursor()
for label, frame in items:
c = config_map[label]
# for interface compatibility with StoreXLSX, where label can be None
if label is STORE_LABEL_DEFAULT:
label = 'None'
else:
label = config_map.default.label_encode(label)
self._frame_to_table(
frame=frame,
label=label,
cursor=cursor,
include_columns=c.include_columns,
include_index=c.include_index,
# store_filter=store_filter
)
conn.commit()
def write(self,
items: tp.Iterable[tp.Tuple[tp.Hashable, Frame]],
*,
config: StoreConfigMapInitializer = None,
# store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT
) -> None:
config_map = StoreConfigMap.from_initializer(config)
with WarningsSilent():
import tables
# silence: DeprecationWarning: `np.typeDict` is a deprecated alias for `np.sctypeDict`.
with tables.open_file(self._fp, mode='w') as file, WarningsSilent():
# silence NaturalNameWarning: object name is not a valid Python identifier:
for label, frame in items:
c = config_map[label]
label = config_map.default.label_encode(label)
# should all tables be under a common group?
field_names, dtypes = self.get_field_names_and_dtypes(
frame=frame,
include_index=c.include_index,
include_index_name=True,
include_columns=c.include_columns,
include_columns_name=False,
)
# Must set pos to have stable position
description = {}
for i, (k, v) in enumerate(zip(field_names, dtypes)):
if v == object:
raise RuntimeError('cannot store object dtypes in HDF5')
description[k] = tables.Col.from_dtype(v, pos=i)
table = file.create_table('/', # create off root from sring
name=label,
description=description,
expectedrows=len(frame),
)
values = self._get_row_iterator(frame=frame,
include_index=c.include_index)
table.append(tuple(values()))
table.flush()
def write(self,
items: tp.Iterable[tp.Tuple[str, Frame]],
*,
config: StoreConfigMapInitializer = None) -> None:
config_map = StoreConfigMap.from_initializer(config)
with zipfile.ZipFile(self._fp, 'w', zipfile.ZIP_DEFLATED) as zf:
for label, frame in items:
c = config_map[label]
dst = BytesIO()
# call from class to explicitly pass self as frame
frame.to_parquet(dst,
include_index=c.include_index,
include_columns=c.include_columns)
dst.seek(0)
# this will write it without a container
zf.writestr(label + self._EXT_CONTAINED, dst.read())
def __init__(
self,
items: IteratorFrameItems,
*,
name: NameType = None,
config: StoreConfigMapInitializer = None,
max_workers: tp.Optional[int] = None,
chunksize: int = 1,
use_threads: bool = False,
):
self._items = items # might be a generator!
self._name = name
self._config = StoreConfigMap.from_initializer(config)
self._max_workers = max_workers
self._chunksize = chunksize
self._use_threads = use_threads
def write(
self,
items: tp.Iterable[tp.Tuple[tp.Optional[str], Frame]],
*,
config: StoreConfigMapInitializer = None
# include_index: bool = True,
# include_columns: bool = True,
# store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT
) -> None:
config_map = StoreConfigMap.from_initializer(config)
import tables
with tables.open_file(self._fp, mode='w') as file:
for label, frame in items:
c = config_map[label]
# should all tables be under a common group?
field_names, dtypes = self.get_field_names_and_dtypes(
frame=frame,
include_index=c.include_index,
include_columns=c.include_columns)
# Must set pos to have stable position
description = {}
for i, (k, v) in enumerate(zip(field_names, dtypes)):
if v == object:
raise RuntimeError(
'cannot store object dtypes in HDF5')
description[k] = tables.Col.from_dtype(v, pos=i)
table = file.create_table(
'/', # create off root from sring
name=label,
description=description,
expectedrows=len(frame),
)
values = self._get_row_iterator(frame=frame,
include_index=c.include_index)
table.append(tuple(values()))
table.flush()
def write(self,
items: tp.Iterable[tp.Tuple[str, Frame]],
config: StoreConfigMapInitializer = None) -> None:
# will create default from None, will pass let a map pass through
config_map = StoreConfigMap.from_initializer(config)
with zipfile.ZipFile(self._fp, 'w', zipfile.ZIP_DEFLATED) as zf:
for label, frame in items:
c = config_map[label]
dst = StringIO()
# call from class to explicitly pass self as frame
self.__class__._EXPORTER(frame,
dst,
include_index=c.include_index,
include_columns=c.include_columns)
dst.seek(0)
# this will write it without a container
zf.writestr(label + self._EXT_CONTAINED, dst.read())
def write(
self,
items: tp.Iterable[tp.Tuple[tp.Optional[str], Frame]],
*,
config: StoreConfigMapInitializer = None,
# include_index: bool = True,
# include_columns: bool = True,
# format_index: tp.Optional[tp.Dict[str, tp.Any]] = None,
# format_columns: tp.Optional[tp.Dict[str, tp.Any]] = None,
# merge_hierarchical_labels: bool = True,
store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT
) -> None:
'''
Args:
store_filter: a dictionary of objects to string, enabling replacement of NaN and None values when writng to XLSX.
'''
# format_data: tp.Optional[tp.Dict[tp.Hashable, tp.Dict[str, tp.Any]]]
# format_data: dictionary of dictionaries, keyed by column label, that contains dictionaries of XlsxWriter format specifications.
# will create default from None, will pass let a map pass through
config_map = StoreConfigMap.from_initializer(config)
import xlsxwriter
wb = xlsxwriter.Workbook(self._fp)
for label, frame in items:
c = config_map[label]
format_columns = self._get_format_or_default(wb, c.format_columns)
format_index = self._get_format_or_default(wb, c.format_index)
ws = wb.add_worksheet(label)
self._frame_to_worksheet(
frame,
ws,
format_columns=format_columns,
format_index=format_index,
include_index=c.include_index,
include_columns=c.include_columns,
merge_hierarchical_labels=c.merge_hierarchical_labels,
store_filter=store_filter)
wb.close()
def _from_store(
cls,
store: Store,
*,
config: StoreConfigMapInitializer = None,
max_workers: tp.Optional[int] = None,
chunksize: int = 1,
use_threads: bool = False,
) -> 'Batch':
config_map = StoreConfigMap.from_initializer(config)
items = ((label, store.read(label, config=config_map[label]))
for label in store.labels(config=config_map))
return cls(
items,
config=config,
max_workers=max_workers,
chunksize=chunksize,
use_threads=use_threads,
)
def read_many(
self,
labels: tp.Iterable[tp.Hashable],
*,
config: StoreConfigMapInitializer = None,
container_type: tp.Type[Frame] = Frame,
) -> tp.Iterator[Frame]:
config_map = StoreConfigMap.from_initializer(config)
sqlite3.register_converter('BOOLEAN', lambda x: x == self._BYTES_ONE)
with sqlite3.connect(self._fp,
detect_types=sqlite3.PARSE_DECLTYPES) as conn:
for label in labels:
c = config_map[label]
if label is STORE_LABEL_DEFAULT:
label_encoded = 'None'
name = None
else:
label_encoded = config_map.default.label_encode(label)
name = label
query = f'SELECT * from "{label_encoded}"'
yield tp.cast(
Frame,
container_type.from_sql(
query=query,
connection=conn,
index_depth=c.index_depth,
index_constructors=c.index_constructors,
columns_depth=c.columns_depth,
columns_select=c.columns_select,
columns_constructors=c.columns_constructors,
dtypes=c.dtypes,
name=name,
consolidate_blocks=c.consolidate_blocks))
def write(
self,
items: tp.Iterable[tp.Tuple[tp.Optional[str], Frame]],
*,
config: StoreConfigMapInitializer = None,
store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT
) -> None:
'''
Args:
store_filter: a dictionary of objects to string, enabling replacement of NaN and None values when writng to XLSX.
'''
# format_data: tp.Optional[tp.Dict[tp.Hashable, tp.Dict[str, tp.Any]]]
# format_data: dictionary of dictionaries, keyed by column label, that contains dictionaries of XlsxWriter format specifications.
# will create default from None, will pass let a map pass through
config_map = StoreConfigMap.from_initializer(config)
import xlsxwriter
# NOTE: can supply second argument: {'default_date_format': 'dd/mm/yy'}
wb = xlsxwriter.Workbook(self._fp, {'remove_timezone': True})
for label, frame in items:
c = config_map[label]
# NOTE: this must be called here, as we need the workbook been assigning formats, and we need to get a config per label
format_columns = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.label, ))
format_index = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.label, ))
format_date = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.date, ))
format_datetime = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.datetime, ))
format_columns_date = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.label, FormatDefaults.date))
format_columns_datetime = FormatDefaults.get_format_or_default(
wb,
format_funcs=(
FormatDefaults.label,
FormatDefaults.datetime,
))
format_index_date = FormatDefaults.get_format_or_default(
wb, format_funcs=(FormatDefaults.label, FormatDefaults.date))
format_index_datetime = FormatDefaults.get_format_or_default(
wb,
format_funcs=(
FormatDefaults.label,
FormatDefaults.datetime,
))
ws = wb.add_worksheet(label)
self._frame_to_worksheet(
frame,
ws,
format_columns=format_columns,
format_index=format_index,
format_date=format_date,
format_datetime=format_datetime,
format_columns_date=format_columns_date,
format_columns_datetime=format_columns_datetime,
format_index_date=format_index_date,
format_index_datetime=format_index_datetime,
include_index=c.include_index,
include_index_name=c.include_index_name,
include_columns=c.include_columns,
include_columns_name=c.include_columns_name,
merge_hierarchical_labels=c.merge_hierarchical_labels,
store_filter=store_filter)
wb.close()
def __init__(
self,
frames: tp.Optional[tp.Iterable[tp.Union[Frame,
tp.Type[FrameDeferred]]]],
*,
index: IndexInitializer,
index_constructor: IndexConstructor = None,
name: NameType = NAME_DEFAULT,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
own_index: bool = False,
own_data: bool = False,
):
'''
Default Bus constructor.
{args}
'''
if max_persist is not None:
# use an (ordered) dictionary to give use an ordered set, simply pointing to None for all keys
self._last_accessed: tp.Dict[tp.Hashable, None] = {}
if own_index:
self._index = index #type: ignore
else:
self._index = index_from_optional_constructor(
index,
default_constructor=Index,
explicit_constructor=index_constructor)
count = len(self._index)
frames_array: np.ndarray
if frames is None:
if store is None:
raise ErrorInitBus(
'Cannot initialize a :obj:`Bus` with neither `frames` nor `store`.'
)
self._values_mutable = np.full(count,
FrameDeferred,
dtype=DTYPE_OBJECT)
self._loaded = np.full(count, False, dtype=DTYPE_BOOL)
self._loaded_all = False
else:
if frames.__class__ is np.ndarray:
if frames.dtype != DTYPE_OBJECT: #type: ignore
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {frames.dtype}'
) #type: ignore
frames_array = frames
load_array = False
else:
if own_data:
raise ErrorInitBus(
'Cannot use `own_data` when not supplying an array.')
frames_array = np.empty(count, dtype=DTYPE_OBJECT)
load_array = True
self._loaded = np.empty(count, dtype=DTYPE_BOOL)
# do a one time iteration of series
for i, (label, value) in enumerate(
zip_longest(
index,
frames,
fillvalue=ZIP_LONGEST_DEFAULT,
)):
if label is ZIP_LONGEST_DEFAULT or value is ZIP_LONGEST_DEFAULT:
raise ErrorInitBus(
'frames and index are not of equal length')
if load_array:
frames_array[i] = value
if value is FrameDeferred:
self._loaded[i] = False
elif isinstance(value, Frame): # permit FrameGO?
if max_persist is not None:
self._last_accessed[label] = None
self._loaded[i] = True
else:
raise ErrorInitBus(
f'supplied {value.__class__} is not a Frame or FrameDeferred.'
)
self._loaded_all = self._loaded.all()
if own_data or load_array:
self._values_mutable = frames_array
else:
self._values_mutable = frames_array.copy()
self._values_mutable.flags.writeable = True
# self._index = index
self._name = None if name is NAME_DEFAULT else name
self._store = store
# Not handling cases of max_persist being greater than the length of the Series (might floor to length)
if max_persist is not None and max_persist < self._loaded.sum():
raise ErrorInitBus(
'max_persist cannot be less than the number of already loaded Frames'
)
self._max_persist = max_persist
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def read_many(
self,
labels: tp.Iterable[tp.Hashable],
*,
config: StoreConfigMapInitializer = None,
store_filter: tp.Optional[StoreFilter] = STORE_FILTER_DEFAULT,
container_type: tp.Type[Frame] = Frame,
) -> tp.Iterator[Frame]:
config_map = StoreConfigMap.from_initializer(config)
wb = self._load_workbook(self._fp)
for label in labels:
c = config_map[label]
index_depth = c.index_depth
index_name_depth_level = c.index_name_depth_level
index_constructors = c.index_constructors
columns_depth = c.columns_depth
columns_name_depth_level = c.columns_name_depth_level
columns_constructors = c.columns_constructors
trim_nadir = c.trim_nadir
skip_header = c.skip_header
skip_footer = c.skip_footer
dtypes = c.dtypes
consolidate_blocks = c.consolidate_blocks
if label is STORE_LABEL_DEFAULT:
ws = wb[wb.sheetnames[0]]
name = None # do not set to default sheet name
else:
label_encoded = config_map.default.label_encode(label)
ws = wb[label_encoded]
name = label # set name to the un-encoded hashable
if ws.max_column <= 1 or ws.max_row <= 1:
# https://openpyxl.readthedocs.io/en/stable/optimized.html
# says that some clients might not report correct dimensions
ws.calculate_dimension()
max_column = ws.max_column
max_row = ws.max_row
# adjust for downward shift for skipping header, then reduce for footer; at this value and beyond we stop
last_row_count = max_row - skip_header - skip_footer
index_values: tp.List[tp.Any] = []
columns_values: tp.List[tp.Any] = []
data = []
apex_rows = []
if trim_nadir:
mask = np.full((last_row_count, max_column), False)
for row_count, row in enumerate(ws.iter_rows(max_row=max_row),
start=-skip_header):
if row_count < 0:
continue # due to skip header; preserves comparison to columns_depth
if row_count >= last_row_count:
break
if trim_nadir:
row_data: tp.Sequence[tp.Any] = []
for col_count, cell in enumerate(row):
if store_filter is None:
value = cell.value
else:
value = store_filter.to_type_filter_element(
cell.value)
if value is None: # NOTE: only checking None, not np.nan
mask[row_count, col_count] = True
row_data.append(value) # type: ignore
if not row_data:
# NOTE: there might be scenarios where there are empty ``row`` iterables that still increment the row_count; we cannot generate these directly for test
mask[row_count] = True #pragma: no cover
else:
if store_filter is None:
row_data = tuple(cell.value for cell in row)
else: # only need to filter string values, but probably too expensive to pre-check
row_data = tuple(
store_filter.to_type_filter_element(cell.value)
for cell in row)
if row_count <= columns_depth - 1:
apex_rows.append(row_data[:index_depth])
if columns_depth == 1:
columns_values.extend(row_data[index_depth:])
elif columns_depth > 1:
columns_values.append(row_data[index_depth:])
continue
if index_depth == 0:
data.append(row_data)
elif index_depth == 1:
index_values.append(row_data[0])
data.append(row_data[1:])
else:
index_values.append(row_data[:index_depth])
data.append(row_data[index_depth:])
#-----------------------------------------------------------------------
# Trim all-empty trailing rows created from style formatting GH#146. As the wb is opened in read-only mode, reverse iterating on the wb is not an option, nor is direct row access by integer
if trim_nadir:
# NOTE: `mask` is all data, while `data` is post index/columns extraction; this means that if a non-None label is found, the row/column will not be trimmed.
row_mask = mask.all(axis=1)
row_trim_start = array1d_to_last_contiguous_to_edge(
row_mask) - columns_depth
if row_trim_start < len(row_mask) - columns_depth:
data = data[:row_trim_start]
if index_depth > 0: # this handles depth 1 and greater
index_values = index_values[:row_trim_start]
col_mask = mask.all(axis=0)
col_trim_start = array1d_to_last_contiguous_to_edge(
col_mask) - index_depth
if col_trim_start < len(col_mask) - index_depth:
data = (r[:col_trim_start] for r in data) #type: ignore
if columns_depth == 1:
columns_values = columns_values[:col_trim_start]
if columns_depth > 1:
columns_values = (r[:col_trim_start]
for r in columns_values
) #type: ignore
#-----------------------------------------------------------------------
# continue with Index and Frame creation
index_name = None if columns_depth == 0 else apex_to_name(
rows=apex_rows,
depth_level=index_name_depth_level,
axis=0,
axis_depth=index_depth)
# index: tp.Optional[IndexBase] = None
if index_depth <= 1:
index_default_constructor = partial(Index, name=index_name)
else: # > 1
index_default_constructor = partial(
IndexHierarchy.from_labels,
name=index_name,
continuation_token=None, # NOTE: needed
)
index, own_index = index_from_optional_constructors(
index_values,
depth=index_depth,
default_constructor=index_default_constructor,
explicit_constructors=index_constructors, # cannot supply name
)
columns_name = None if index_depth == 0 else apex_to_name(
rows=apex_rows,
depth_level=columns_name_depth_level,
axis=1,
axis_depth=columns_depth)
# columns: tp.Optional[IndexBase] = None
# own_columns = False
if columns_depth <= 1:
columns_default_constructor = partial(
container_type._COLUMNS_CONSTRUCTOR,
name=columns_name,
)
elif columns_depth > 1:
columns_default_constructor = partial(
container_type._COLUMNS_HIERARCHY_CONSTRUCTOR.from_labels,
name=columns_name,
continuation_token=None, # NOTE: needed, not the default
)
columns_values = zip(*columns_values) #type: ignore
columns, own_columns = index_from_optional_constructors(
columns_values,
depth=columns_depth,
default_constructor=columns_default_constructor,
explicit_constructors=columns_constructors, # cannot supply name
)
yield container_type.from_records(
data,
index=index,
columns=columns,
dtypes=dtypes,
own_index=own_index,
own_columns=own_columns,
name=name,
consolidate_blocks=consolidate_blocks)
wb.close()
def read_many(
self,
labels: tp.Iterable[tp.Hashable],
*,
config: StoreConfigMapInitializer = None,
container_type: tp.Type[Frame] = Frame,
) -> tp.Iterator[Frame]:
config_map = StoreConfigMap.from_initializer(config)
multiprocess: bool = config_map.default.read_max_workers is not None
constructor: FrameConstructor = self._container_type_to_constructor(
container_type)
if not multiprocess:
yield from self._read_many_single_thread(
labels=labels,
config_map=config_map,
constructor=constructor,
container_type=container_type,
)
return
count_cache: int = 0
if self._weak_cache:
count_labels: int = 0
results: tp.Dict[tp.Hashable, tp.Optional[Frame]] = {}
for label in labels:
count_labels += 1
cache_lookup = self._weak_cache.get(label,
NOT_IN_CACHE_SENTINEL)
if cache_lookup is not NOT_IN_CACHE_SENTINEL:
results[label] = self._set_container_type(
cache_lookup, container_type)
count_cache += 1
else:
results[label] = None
def results_items() -> IteratorItemsLabelOptionalFrame:
yield from results.items()
else:
labels = list(labels)
def results_items() -> IteratorItemsLabelOptionalFrame:
for label in labels:
yield label, None
# Avoid spinning up a process pool if all requested labels had weakrefs
if count_cache and count_cache == count_labels:
for _, frame in results_items():
assert frame is not None # mypy
yield frame
return
def gen() -> tp.Iterator[PayloadBytesToFrame]:
'''
This method is synchronized with the following `for label in results_items` loop, as they both share the same necessary & initial condition: `if cached_frame is not None`.
'''
with zipfile.ZipFile(self._fp) as zf:
for label, cached_frame in results_items():
if cached_frame is not None:
continue
c: StoreConfig = config_map[label]
label_encoded: str = config_map.default.label_encode(label)
src: bytes = zf.read(label_encoded + self._EXT_CONTAINED)
yield PayloadBytesToFrame( # pylint: disable=no-value-for-parameter
src=src,
name=label,
config=c.to_store_config_he(),
constructor=constructor,
)
chunksize = config_map.default.read_chunksize
with ProcessPoolExecutor(
max_workers=config_map.default.read_max_workers) as executor:
frame_gen = executor.map(self._payload_to_frame,
gen(),
chunksize=chunksize)
for label, cached_frame in results_items():
if cached_frame is not None:
yield cached_frame
else:
frame = next(frame_gen)
# Newly read frame, add it to our weak_cache
self._weak_cache[label] = frame
yield frame