def testDictWithList(self):
DataMiner().data_type = DataTypes.DictWithList
self.store_key()
print(PersistentObjectStorage().storage_object)
processor = DictProcessing(PersistentObjectStorage().storage_object)
processor.simplify()
self.assertIn(
"'a': {'d': {'e': {'%s': [" % DataMiner().key,
str(PersistentObjectStorage().storage_object),
)
def testCallNoDebug(self):
DataMiner().store_arg_debug_metadata = False
OwnClass.get_num = ObjectStorage.decorator_plain(OwnClass.get_num)
test_obj = OwnClass(3)
obj_before_4 = test_obj.get_num(4)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
obj_after_4 = test_obj.get_num(4)
obj_after_4_meta = DataMiner().metadata
self.assertEqual(obj_before_4, obj_after_4)
self.assertIsInstance(obj_after_4, int)
self.assertEqual(
obj_after_4_meta.get(DataMiner().METADATA_ARG_DEBUG_KEY), None)
def test_key_exact_miner_exception(self):
DataMiner().read_key_exact = True
self.assertRaises(
PersistentStorageException,
PersistentObjectStorage().read,
["y"] + self.keys,
)
def test_dict_data(self):
DataMiner().data_type = DataTypes.Dict
DataMiner().key = "first-key"
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={})
DataMiner().key = "second-key"
PersistentObjectStorage().store(keys=self.keys,
values="y",
metadata={})
self.assertIn("first-key",
PersistentObjectStorage().storage_object["a"]["b"])
self.assertIn("second-key",
PersistentObjectStorage().storage_object["a"]["b"])
self.assertEqual("y", PersistentObjectStorage()[self.keys])
DataMiner().key = "first-key"
self.assertEqual("x", PersistentObjectStorage()[self.keys])
def testValue(self):
DataMiner().data_type = DataTypes.Value
self.store_key()
print(PersistentObjectStorage().storage_object)
processor = DictProcessing(PersistentObjectStorage().storage_object)
processor.simplify()
self.assertIn(
"'a': {'d': {'e': {'metadata'",
str(PersistentObjectStorage().storage_object),
)
def test_strategy(self):
DataMiner().key_stategy_cls = StorageKeysInspectSimple
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={})
self.simple_return("ahoj")
self.simple_return([1, 2, 3])
PersistentObjectStorage().dump()
PersistentObjectStorage().storage_object = {}
DataMiner().key_stategy_cls = StorageKeysInspectDefault
PersistentObjectStorage().mode = StorageMode.read
PersistentObjectStorage().load()
self.assertEqual("x", PersistentObjectStorage()[self.keys])
self.assertEqual(
PersistentObjectStorage().metadata.get(
PersistentObjectStorage().key_inspect_strategy_key),
StorageKeysInspectSimple.__name__,
)
self.assertEqual(StorageKeysInspectSimple, DataMiner().key_stategy_cls)
def apply_fn():
"""
This function is used when installed as sitecustomize.py script
to enable replacing system, please set env vars RESPONSE_FILE
REPLACEMENT_FILE, see doc string of this file
"""
# file name of storage file
storage_file = os.getenv(ENV_STORAGE_FILE)
# file name of replaces for updated import system
replacement_file = os.getenv(ENV_REPLACEMENT_FILE)
if_latency = os.getenv(ENV_APPLY_LATENCY)
replacement_var = os.getenv(ENV_REPLACEMENT_NAME, REPLACE_DEFAULT_KEY)
debug_print(
f"You have patched version of your python by requre project "
f"(python {sys.version_info.major}.{sys.version_info.minor}, {__file__}) "
)
if not (storage_file and replacement_file):
debug_print(f"\tYou have to set {ENV_STORAGE_FILE} and "
f"{ENV_REPLACEMENT_FILE} env variables to work properly")
else:
if not os.path.exists(replacement_file):
raise FileExistsError(
f"{replacement_file} has to exist to work properly "
f"(python file with replacements definition)")
if if_latency:
debug_print("Use latency for function calls")
DataMiner().use_latency = True
PersistentObjectStorage().storage_file = storage_file
spec = importlib.util.spec_from_file_location("replacements",
replacement_file)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if hasattr(module, replacement_var):
replacement = getattr(module, replacement_var)
debug_print(f"Replaces: {replacement}")
if isinstance(replacement, UpgradeImportSystem):
debug_print(
f"{replacement_var} is {UpgradeImportSystem.__name__} object"
)
elif isinstance(replacement, list):
debug_print(
f"{replacement_var} is list of replacements, apply upgrading"
)
upgrade_import_system(filters=replacement)
else:
raise_error(
126, f"Bad type of {replacement_var}, see documentation")
else:
raise_error(
125,
f"in {replacement_file} there is not defined '{replacement_var}' variable",
)
# register dump command, when python finish
atexit.register(PersistentObjectStorage().dump)
def execute(cls, keys: list, func: Callable, *args, **kwargs) -> Any:
"""
Class method to store or read object from persistent storage
with explicit set of *args, **kwargs parameters to use as keys
:param keys: list of keys used as parameter
:param func: original function
:param args: parameters of original function
:param kwargs: parameters of original function
:return: output of called func
"""
object_storage = cls(store_keys=keys)
if object_storage.persistent_storage.do_store(keys):
time_before = original_time()
response = func(*args, **kwargs)
time_after = original_time()
metadata: Dict = {
DataMiner().LATENCY_KEY:
time_after - time_before,
DataMiner().METADATA_CALLER_LIST:
StorageKeysInspectFull.get_base_keys(func),
}
if DataMiner().store_arg_debug_metadata:
args_clean = [
f"'{x}'" if isinstance(x, str) else str(x) for x in args
]
kwargs_clean = [
f"""{k}={f"'{v}'" if isinstance(v, str) else str(v)}"""
for k, v in kwargs.items()
]
caller = f"{func.__name__}({', '.join(args_clean + kwargs_clean)})"
metadata[DataMiner().METADATA_ARG_DEBUG_KEY] = caller
object_storage.write(response, metadata)
logger.debug(f"WRITE Keys: {keys} -> {response}")
return response
else:
response = object_storage.read()
logger.debug(f"READ Keys: {keys} -> {response}")
return response
def _copy_logic(cls, pers_storage: PersistentObjectStorage, pathname: str,
keys: list) -> None:
"""
Internal function. Copy files to or back from persisten storage
It will create tar archive with tar_compression and stores it to Persistent Storage
"""
logger.debug(f"Copy files {pathname} -> {keys}")
logger.debug(f"Persistent Storage mode: {pers_storage.mode}")
original_cwd = os.getcwd()
if pers_storage.do_store(keys=cls.basic_ps_keys + keys):
try:
artifact_name = os.path.basename(pathname)
artifact_path = os.path.dirname(pathname)
os.chdir(artifact_path)
with BytesIO() as fileobj:
with tarfile.open(mode=f"x:{cls.tar_compression}",
fileobj=fileobj) as tar_store:
tar_store.add(name=artifact_name)
fileobj.seek(0)
metadata = {DataMiner().LATENCY_KEY: 0}
pers_storage.store(
keys=cls.basic_ps_keys + keys,
values=fileobj.read(),
metadata=metadata,
)
finally:
os.chdir(original_cwd)
else:
value = pers_storage[cls.basic_ps_keys + keys]
with BytesIO(value) as fileobj:
with tarfile.open(mode=f"r:{cls.tar_compression}",
fileobj=fileobj) as tar_store:
tarinfo_1st_member = tar_store.getmembers()[0]
if tarinfo_1st_member.isfile():
with open(pathname, mode="wb") as output_file:
output_file.write(
tar_store.extractfile(
tarinfo_1st_member).read())
else:
for tar_item in tar_store.getmembers():
# we have to modify path of files to remove topdir
if len(tar_item.name.split(os.path.sep, 1)) > 1:
tar_item.name = tar_item.name.split(
os.path.sep, 1)[1]
else:
tar_item.name = "."
try:
tar_store.extract(tar_item, path=pathname)
except IOError:
# rewrite readonly files if necessary
os.remove(os.path.join(pathname,
tar_item.name))
tar_store.extract(tar_item, path=pathname)
def testCallDebug(self):
OwnClass.get_num = ObjectStorage.decorator_plain(OwnClass.get_num)
test_obj = OwnClass(3)
obj_before_4 = test_obj.get_num(4)
obj_before_5 = test_obj.get_num(5, extended=True)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
obj_after_4 = test_obj.get_num(4)
obj_after_4_meta = DataMiner().metadata
obj_after_5 = test_obj.get_num(5, extended=True)
obj_after_5_meta = DataMiner().metadata
self.assertEqual(obj_before_4, obj_after_4)
self.assertIsInstance(obj_after_4, int)
self.assertRegex(obj_after_4_meta[DataMiner().METADATA_ARG_DEBUG_KEY],
"get_num(.*, 4)")
self.assertEqual(obj_before_5, obj_after_5)
self.assertIsInstance(obj_after_5, str)
self.assertRegex(
obj_after_5_meta[DataMiner().METADATA_ARG_DEBUG_KEY],
"get_num(.*, 5, extended=True)",
)
def test_applied(self):
delta = 0.05
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={})
time.sleep(0.2)
PersistentObjectStorage().store(keys=self.keys,
values="y",
metadata={})
self.assertAlmostEqual(0.2,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
time_begin = time.time()
PersistentObjectStorage().read(self.keys)
time_end = time.time()
self.assertAlmostEqual(0, time_end - time_begin, delta=delta)
time_begin = time.time()
PersistentObjectStorage().read(self.keys)
time_end = time.time()
self.assertAlmostEqual(0.2, time_end - time_begin, delta=delta)
def setUp(self) -> None:
DataMiner().key_stategy_cls = StorageKeysInspectSimple
if self.get_datafile_filename().exists():
# if already exists, do not regenerate test file, what is stored inside tests dir
PersistentObjectStorage().storage_file = str(
self.get_datafile_filename())
else:
# store them to path where Persistent volume is mounted
PersistentObjectStorage().storage_file = str(
self.get_datafile_filename(path_prefix=Path("/tmp") /
TEST_DATA_DIR))
self.steve = SteveJobs()
self.steve.config.command_handler = RunCommandType.local
self.steve.config.command_handler_work_dir = "/tmp/hello-world"
def test_value_data(self):
DataMiner().data_type = DataTypes.Value
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={})
PersistentObjectStorage().store(keys=self.keys,
values="y",
metadata={})
self.assertEqual(
2, len(PersistentObjectStorage().storage_object["a"]["b"]))
self.assertEqual("y", PersistentObjectStorage()[self.keys])
self.assertEqual(
2, len(PersistentObjectStorage().storage_object["a"]["b"]))
self.assertEqual("y", PersistentObjectStorage()[self.keys])
self.assertEqual(
2, len(PersistentObjectStorage().storage_object["a"]["b"]))
def test_generic(self):
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={"test_meta": "yes"})
PersistentObjectStorage().metadata = {"rpms": ["package1", "package2"]}
PersistentObjectStorage().dump()
PersistentObjectStorage().storage_object = {}
PersistentObjectStorage().load()
self.assertIn(self.keys, PersistentObjectStorage())
self.assertEqual(DataMiner().metadata["test_meta"], "yes")
self.assertEqual(
PersistentObjectStorage().metadata.get(
PersistentObjectStorage().key_inspect_strategy_key),
StorageKeysInspectDefault.__name__,
)
self.assertEqual(PersistentObjectStorage().metadata.get("rpms"),
["package1", "package2"])
def tearDown(self):
DataMiner().read_key_exact = False
def tearDown(self):
DataMiner().use_latency = False
def setUp(self):
super().setUp()
DataMiner().current_time = time.time()
DataMiner().use_latency = True
def test_strategy_default_cls(self):
self.test_strategy()
PersistentObjectStorage().load()
DataMiner().key_stategy_cls = StorageKeysInspectDefault
self.assertEqual("ahoj", self.simple_return("nonsense"))
def test_strategy_API_cls(self):
self.test_strategy()
PersistentObjectStorage().load()
DataMiner().key_stategy_cls = StorageKeysInspect
self.assertRaises(NotImplementedError, self.simple_return, "nonsense")
def tearDown(self):
# return it to default type
DataMiner().data_type = DataTypes.List
super().tearDown()
def tearDown(self) -> None:
DataMiner().store_arg_debug_metadata = False
super().tearDown()
def get_base_keys(func: Callable) -> List[Any]:
return DataMiner().key_stategy_cls.get_base_keys(func)
def tearDown(self):
DataMiner().data_type = DataTypes.List
def test_key_exact_miner(self):
DataMiner().read_key_exact = True
output = PersistentObjectStorage().read(self.keys)
self.assertEqual("x", output)
def setUp(self) -> None:
super().setUp()
DataMiner().store_arg_debug_metadata = True
def test_strategy_default_cls(self):
self.test_strategy()
PersistentObjectStorage().load()
DataMiner().key_stategy_cls = StorageKeysInspectDefault
self.assertRaises(PersistentStorageException, self.simple_return,
"nonsense")
def test_default(self):
self.assertEqual(DataTypes.List, DataMiner().data_type)
def setUp(self):
super().setUp()
DataMiner().current_time = time.time()
def test_latency(self):
delta = 0.05
PersistentObjectStorage().store(keys=self.keys,
values="x",
metadata={})
self.assertAlmostEqual(0,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
# Try to set some custom metadata
DataMiner().data.metadata = {"random": "data"}
time.sleep(0.1)
PersistentObjectStorage().store(keys=self.keys,
values="y",
metadata={})
self.assertAlmostEqual(0.1,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
time.sleep(0.2)
PersistentObjectStorage().store(keys=self.keys,
values="z",
metadata={})
self.assertAlmostEqual(0.2,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
PersistentObjectStorage().read(self.keys)
self.assertAlmostEqual(0,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
# check custom metadata
self.assertEqual("data", DataMiner().metadata["random"])
PersistentObjectStorage().read(self.keys)
self.assertAlmostEqual(0.1,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
# check if custom metadata are not everywhere
self.assertNotIn("random", DataMiner().metadata)
self.assertIn("latency", DataMiner().metadata)
PersistentObjectStorage().read(self.keys)
self.assertAlmostEqual(0.2,
DataMiner().metadata[DataMiner().LATENCY_KEY],
delta=delta)
def setUp(self):
super().setUp()
DataMiner().key_stategy_cls = StorageKeysInspectSimple