def get_current_notebook_code():
path = get_current_notebook_path()
path = path.replace("\\", "/")
s = fr'jupyter nbconvert --to script "{path}"'
execute(s)
with open(path.replace(".ipynb", ".py"), "r") as f:
code = f.read()
remove_path(path.replace(".ipynb", ".py"))
return code
def assert_logs(self, reset=False):
if reset and os.path.exists(self.path):
remove_path(self.path)
if not os.path.exists(self.path):
logger.info(f"Logs not found: {self.path}. Initializing...")
self._flush(self.path)
else:
with tempfile.TemporaryDirectory() as temp_dir:
self._flush(temp_dir)
self._assert_equal_directory_contents(temp_dir, self.path)
def test_pickle_db_state_provider():
state_provider = PickleDBStateProvider("state.pickledb")
state_provider.set_state(16)
assert state_provider.get_state() == 16
PickleDBStateProvider("state.pickledb").set_state(17)
assert PickleDBStateProvider("state.pickledb").get_state() == 17
with PickleDBStateProvider("state.pickledb") as sp:
assert sp.get_state() == 17
sp.set_state(18)
assert sp.get_state() == 18
print(state_provider.get_state())
# clean up
from utils_ak.os import remove_path
remove_path("state.pickledb")
def test_pandas_split_combine_etl():
import time
from utils_ak.time import cast_dt
from utils_ak.os import remove_path
df = pd.DataFrame(
list(range(100)),
index=pd.date_range(cast_dt("2020.01.01"), periods=100, freq="1d"),
)
etl = PandasSplitCombineETL(
path="test-data/",
extension=".parquet",
key_func=lambda df: pd.Series(df.index, index=df.index).apply(
lambda dt: cast_str(dt, "%Y%m")),
)
etl.split_and_load(df)
print(etl.extract_and_combine())
remove_path("test-data/")
def leave_unstable_state(
self): # Call me if you think that operation ended
logger.debug("Leaving unstable state")
remove_path(self.canary_path)
def __exit__(self, exc_type, exc_val, exc_tb):
remove_path(self.canary_fn)
def _recover(self):
if os.path.exists(self.canary_fn):
self.canary_db = pickledb.load(self.canary_fn, auto_dump=False)
self.set_state(self.canary_db.get(self.key))
remove_path(self.canary_fn)
def remove(self):
remove_path(self.path)
def split_and_load(self, combined, merge=True):
if not merge:
remove_path(self.path)
makedirs(self.path)
for key, split in self._split(combined):
self._load(key, split, merge=merge)