def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(f'supplied label {label} is not a string.')
if isinstance(value, Frame):
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
own_data: bool = False,
):
'''
Default Bus constructor.
{args}
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
if max_persist is not None:
# use an (ordered) dictionary to give use an ordered set, simply pointing to None for all keys
self._last_accessed: tp.Dict[str, None] = {}
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if isinstance(value, Frame):
if max_persist is not None:
self._last_accessed[label] = None
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
if own_data:
self._values_mutable = series.values
self._values_mutable.flags.writeable = True
else:
self._values_mutable = series.values.copy()
self._index = series._index
self._name = series._name
self._store = store
# Not handling cases of max_persist being greater than the length of the Series (might floor to length)
if max_persist is not None and max_persist < self._loaded.sum():
raise ErrorInitBus('max_persist cannot be less than the number of already loaded Frames')
self._max_persist = max_persist
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def __init__(self, series: Series, *, store: tp.Optional[Store] = None):
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}'
)
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(
f'supplied label {label} is not a string.')
if isinstance(value, Frame):
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(
f'supplied {value.__class__} is not a Frame or FrameDeferred.'
)
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
self._series = series
self._store = store
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None
):
'''
Args:
config: StoreConfig for handling ``Frame`` construction and exporting from Store.
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(f'supplied label {label} is not a string.')
if isinstance(value, Frame):
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
self._series = series
self._store = store
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if isinstance(value, Frame):
if max_persist is not None:
self._last_accessed[label] = None
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
def __init__(self,
series: Series,
*,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
):
'''
Args:
config: StoreConfig for handling :obj:`Frame` construction and exporting from Store.
max_persist: When loading :obj:`Frame` from a :obj:`Store`, optionally define the maximum number of :obj:`Frame` to remain in the :obj:`Bus`, regardless of the size of the :obj:`Bus`. If more than ``max_persist`` number of :obj:`Frame` are loaded, least-recently loaded :obj:`Frame` will be replaced by ``FrameDeferred``. A ``max_persist`` of 1, for example, permits reading one :obj:`Frame` at a time without ever holding in memory more than 1 :obj:`Frame`.
'''
if series.dtype != DTYPE_OBJECT:
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {series.dtype}')
if max_persist is not None:
self._last_accessed: tp.Dict[str, None] = {}
# do a one time iteration of series
def gen() -> tp.Iterator[bool]:
for label, value in series.items():
if not isinstance(label, str):
raise ErrorInitBus(f'supplied label {label} is not a string.')
if isinstance(value, Frame):
if max_persist is not None:
self._last_accessed[label] = None
yield True
elif value is FrameDeferred:
yield False
else:
raise ErrorInitBus(f'supplied {value.__class__} is not a Frame or FrameDeferred.')
self._loaded = np.fromiter(gen(), dtype=DTYPE_BOOL, count=len(series))
self._loaded_all = self._loaded.all()
self._series = series
self._store = store
# max_persist might be less than the number of Frames already loaded
if max_persist is not None:
self._max_persist = max(max_persist, self._loaded.sum())
else:
self._max_persist = None
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)
def __init__(
self,
frames: tp.Optional[tp.Iterable[tp.Union[Frame,
tp.Type[FrameDeferred]]]],
*,
index: IndexInitializer,
index_constructor: IndexConstructor = None,
name: NameType = NAME_DEFAULT,
store: tp.Optional[Store] = None,
config: StoreConfigMapInitializer = None,
max_persist: tp.Optional[int] = None,
own_index: bool = False,
own_data: bool = False,
):
'''
Default Bus constructor.
{args}
'''
if max_persist is not None:
# use an (ordered) dictionary to give use an ordered set, simply pointing to None for all keys
self._last_accessed: tp.Dict[tp.Hashable, None] = {}
if own_index:
self._index = index #type: ignore
else:
self._index = index_from_optional_constructor(
index,
default_constructor=Index,
explicit_constructor=index_constructor)
count = len(self._index)
frames_array: np.ndarray
if frames is None:
if store is None:
raise ErrorInitBus(
'Cannot initialize a :obj:`Bus` with neither `frames` nor `store`.'
)
self._values_mutable = np.full(count,
FrameDeferred,
dtype=DTYPE_OBJECT)
self._loaded = np.full(count, False, dtype=DTYPE_BOOL)
self._loaded_all = False
else:
if frames.__class__ is np.ndarray:
if frames.dtype != DTYPE_OBJECT: #type: ignore
raise ErrorInitBus(
f'Series passed to initializer must have dtype object, not {frames.dtype}'
) #type: ignore
frames_array = frames
load_array = False
else:
if own_data:
raise ErrorInitBus(
'Cannot use `own_data` when not supplying an array.')
frames_array = np.empty(count, dtype=DTYPE_OBJECT)
load_array = True
self._loaded = np.empty(count, dtype=DTYPE_BOOL)
# do a one time iteration of series
for i, (label, value) in enumerate(
zip_longest(
index,
frames,
fillvalue=ZIP_LONGEST_DEFAULT,
)):
if label is ZIP_LONGEST_DEFAULT or value is ZIP_LONGEST_DEFAULT:
raise ErrorInitBus(
'frames and index are not of equal length')
if load_array:
frames_array[i] = value
if value is FrameDeferred:
self._loaded[i] = False
elif isinstance(value, Frame): # permit FrameGO?
if max_persist is not None:
self._last_accessed[label] = None
self._loaded[i] = True
else:
raise ErrorInitBus(
f'supplied {value.__class__} is not a Frame or FrameDeferred.'
)
self._loaded_all = self._loaded.all()
if own_data or load_array:
self._values_mutable = frames_array
else:
self._values_mutable = frames_array.copy()
self._values_mutable.flags.writeable = True
# self._index = index
self._name = None if name is NAME_DEFAULT else name
self._store = store
# Not handling cases of max_persist being greater than the length of the Series (might floor to length)
if max_persist is not None and max_persist < self._loaded.sum():
raise ErrorInitBus(
'max_persist cannot be less than the number of already loaded Frames'
)
self._max_persist = max_persist
# providing None will result in default; providing a StoreConfig or StoreConfigMap will return an appropriate map
self._config = StoreConfigMap.from_initializer(config)