def test_create_file_content_positional(self):
"""
Similar to test_create_file_content,
but test it also via positional parameters and mixing them
"""
self.create_temp_file()
self.create_file_content("ahoj", self.temp_file)
self.create_file_content("cao", self.temp_file)
PersistentObjectStorage().cassette.dump()
PersistentObjectStorage().cassette.mode = StorageMode.read
self.create_temp_file()
self.create_file_content("first", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
self.create_temp_file()
self.create_file_content("second", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.create_temp_file()
self.assertRaises(Exception,
self.create_file_content,
"third",
target_file=self.temp_file)
def test(self):
"""
Mixing command wrapper with file storage
"""
self.assertEqual(StoreFiles.counter, 0)
self.create_temp_file()
self.create_file_content("ahoj", target_file=self.temp_file)
self.assertEqual(StoreFiles.counter, 1)
self.create_file_content("cao", target_file=self.temp_file)
self.assertEqual(StoreFiles.counter, 2)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
before = str(PersistentObjectStorage().storage_object)
self.create_file_content("ahoj", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
self.create_file_content("cao", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
after = str(PersistentObjectStorage().storage_object)
self.assertGreater(len(before), len(after))
self.assertIn("True", before)
def test_create_dir_content(self):
"""
Check if properly store and restore directory content
"""
self.create_temp_dir()
self.create_dir_content(filename="ahoj",
target_dir=self.temp_dir,
content="ciao")
self.assertIn("ahoj", os.listdir(self.temp_dir))
with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd:
content = fd.read()
self.assertIn("ciao", content)
PersistentObjectStorage().cassette.dump()
PersistentObjectStorage().cassette.mode = StorageMode.read
self.create_temp_dir()
self.create_dir_content(filename="nonsense",
target_dir=self.temp_dir,
content="bad")
self.assertIn("ahoj", os.listdir(self.temp_dir))
self.assertNotIn("nonsense", os.listdir(self.temp_dir))
with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd:
content = fd.read()
self.assertIn("ciao", content)
self.assertNotIn("bad", content)
def test_create_file_content(self):
"""
test if is able store files via decorated function create_file_content
"""
self.assertEqual(StoreFiles.counter, 0)
self.create_temp_file()
self.assertEqual(
"value",
self.create_file_content("ahoj", target_file=self.temp_file))
self.assertEqual(StoreFiles.counter, 1)
self.create_file_content("cao", target_file=self.temp_file)
self.assertEqual(StoreFiles.counter, 2)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
self.create_file_content("first", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
self.create_file_content("second", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.assertRaises(Exception,
self.create_file_content,
"third",
target_file=self.temp_file)
def test_create_file_content_positional(self):
"""
Similar to test_create_file_content,
but test it also via positional parameters and mixing them
"""
self.assertEqual(StoreFiles.counter, 0)
self.create_temp_file()
self.create_file_content("ahoj", self.temp_file)
self.assertEqual(StoreFiles.counter, 1)
self.create_file_content("cao", self.temp_file)
self.assertEqual(StoreFiles.counter, 2)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
self.create_temp_file()
self.create_file_content("first", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
self.create_temp_file()
self.create_file_content("second", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.create_temp_file()
self.assertRaises(Exception,
self.create_file_content,
"third",
target_file=self.temp_file)
def test_create_file_content(self):
"""
test if is able store files via decorated function create_file_content
"""
self.create_temp_file()
self.assertEqual(
"value",
self.create_file_content("ahoj", target_file=self.temp_file))
self.create_file_content("cao", target_file=self.temp_file)
PersistentObjectStorage().cassette.dump()
PersistentObjectStorage().cassette.mode = StorageMode.read
self.create_file_content("first", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
self.create_file_content("second", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.assertRaises(Exception,
self.create_file_content,
"third",
target_file=self.temp_file)
def setUp(self):
self.github_token = os.environ.get("GITHUB_TOKEN")
self.pagure_token = os.environ.get("PAGURE_TOKEN")
self.gitlab_token = os.environ.get("GITLAB_TOKEN") or "some_token"
test_name = self.id() or "all"
persistent_data_file = os.path.join(
PERSISTENT_DATA_PREFIX, f"test_factory_data_{test_name}.yaml")
PersistentObjectStorage().storage_file = persistent_data_file
if PersistentObjectStorage().is_write_mode and not self.github_token:
raise EnvironmentError("please set GITHUB_TOKEN env variables")
if PersistentObjectStorage().is_write_mode and not self.pagure_token:
raise EnvironmentError("please set PAGURE_TOKEN env variables")
if PersistentObjectStorage(
).is_write_mode and not os.environ.get("GITLAB_TOKEN"):
raise EnvironmentError("please set GITLAB_TOKEN env variables")
self.github_service = GithubService(token=self.github_token)
self.pagure_service = PagureService(token=self.pagure_token)
self.gitlab_service = GitlabService(token=self.gitlab_token,
instance_url="https://gitlab.com")
self.custom_instances = [
self.github_service,
self.pagure_service,
self.gitlab_service,
]
def test_run_command_output(self):
"""
check if wrapper returns proper string values in calls
"""
self.file_name = tempfile.mktemp()
with open(self.file_name, "w") as fd:
fd.write("ahoj\n")
output = self.run_command_wrapper(cmd=["cat", self.file_name],
output=True)
self.assertIn("ahoj", output)
PersistentObjectStorage().cassette.dump()
PersistentObjectStorage().cassette.mode = StorageMode.read
with open(self.file_name, "a") as fd:
fd.write("cao\n")
output = self.run_command_wrapper(cmd=["cat", self.file_name],
output=True)
self.assertIn("ahoj", output)
self.assertNotIn("cao", output)
PersistentObjectStorage().cassette.mode = StorageMode.write
output = self.run_command_wrapper(cmd=["cat", self.file_name],
output=True)
self.assertIn("cao", output)
PersistentObjectStorage().cassette.dump()
PersistentObjectStorage().cassette.mode = StorageMode.read
output = self.run_command_wrapper(cmd=["cat", self.file_name],
output=True)
self.assertIn("cao", output)
def test_create_dir_content(self):
"""
Check if properly store and restore directory content
"""
self.assertEqual(StoreFiles.counter, 0)
self.create_temp_dir()
self.create_dir_content(filename="ahoj",
target_dir=self.temp_dir,
content="ciao")
self.assertEqual(StoreFiles.counter, 1)
self.assertIn("ahoj", os.listdir(self.temp_dir))
with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd:
content = fd.read()
self.assertIn("ciao", content)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
self.create_temp_dir()
self.create_dir_content(filename="nonsense",
target_dir=self.temp_dir,
content="bad")
self.assertIn("ahoj", os.listdir(self.temp_dir))
self.assertNotIn("nonsense", os.listdir(self.temp_dir))
with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd:
content = fd.read()
self.assertIn("ciao", content)
self.assertNotIn("bad", content)
def testChangeFile(self):
PersistentObjectStorage().storage_file += ".x"
output = TempFile.mktemp()
self.assertIn(
f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_1",
output,
)
output = TempFile.mktemp()
self.assertIn(
f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_2",
output,
)
PersistentObjectStorage().storage_file += ".y"
self.assertEqual(TempFile.counter, 2)
output = TempFile.mktemp()
self.assertEqual(TempFile.counter, 1)
self.assertIn(
f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_1",
output,
)
output = TempFile.mktemp()
self.assertIn(
f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_2",
output,
)
def test_create_file(self):
"""
Test File storage if file name is return value of function
"""
ofile1 = self.create_file("ahoj")
ofile2 = self.create_file("cao")
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
oofile1 = self.create_file("first")
with open(ofile1, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
with open(oofile1, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
oofile2 = self.create_file("second")
with open(oofile2, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.assertEqual(ofile2, oofile2)
def setUp(self):
super().setUp()
print(PersistentObjectStorage().storage_file)
self.github_token = os.environ.get("GITHUB_TOKEN")
self.pagure_token = os.environ.get("PAGURE_TOKEN")
self.gitlab_token = os.environ.get("GITLAB_TOKEN") or "some_token"
if PersistentObjectStorage().mode == StorageMode.write:
if not self.github_token:
raise EnvironmentError(
"You are in requre write mode, please set GITHUB_TOKEN env variables"
)
if not self.pagure_token:
raise EnvironmentError(
"You are in requre write mode, please set PAGURE_TOKEN env variables"
)
if not os.environ.get("GITLAB_TOKEN"):
raise EnvironmentError(
"You are in requre write mode, please set GITLAB_TOKEN env variables"
)
self.github_service = GithubService(token=self.github_token)
self.pagure_service = PagureService(token=self.pagure_token)
self.gitlab_service = GitlabService(
token=self.gitlab_token, instance_url="https://gitlab.com"
)
self.custom_instances = [
self.github_service,
self.pagure_service,
self.gitlab_service,
]
def test_create_file(self):
"""
File storage where it try to guess what to store, based on *args and **kwargs values
"""
self.create_temp_file()
self.write_to_file("ahoj", self.temp_file)
self.write_to_file("cao", self.temp_file)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
self.create_temp_file()
self.write_to_file("ahoj", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
self.create_temp_file()
self.write_to_file("cao", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.create_temp_file()
self.assertRaises(
Exception, self.create_file_content, "third", target_file=self.temp_file
)
def test(self):
"""
Mixing command wrapper with file storage
"""
self.create_temp_file()
self.create_file_content("ahoj", target_file=self.temp_file)
self.create_file_content("cao", target_file=self.temp_file)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
before = str(PersistentObjectStorage().storage_object)
self.create_file_content("ahoj", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
self.create_file_content("cao", target_file=self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
after = str(PersistentObjectStorage().storage_object)
self.assertGreater(len(before), len(after))
self.assertIn("True", before)
def test_create_file(self):
"""
Test File storage if file name is return value of function
"""
self.assertEqual(StoreFiles.counter, 0)
ofile1 = self.create_file("ahoj")
self.assertEqual(StoreFiles.counter, 1)
ofile2 = self.create_file("cao")
self.assertEqual(StoreFiles.counter, 2)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
oofile1 = self.create_file("first")
with open(ofile1, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
with open(oofile1, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
oofile2 = self.create_file("second")
with open(oofile2, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.assertEqual(ofile2, oofile2)
def test_create_file(self):
"""
File storage where it try to guess what to store, based on *args and **kwargs values
"""
self.assertEqual(StoreFiles.counter, 0)
self.create_temp_file()
self.create_file("ahoj", self.temp_file)
self.assertEqual(StoreFiles.counter, 1)
self.create_file("cao", self.temp_file)
self.assertEqual(StoreFiles.counter, 2)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
StoreFiles.counter = 0
self.create_temp_file()
self.create_file("first", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertIn("ahoj", content)
self.assertNotIn("cao", content)
# mix with positional option
self.create_temp_file()
self.create_file("second", self.temp_file)
with open(self.temp_file, "r") as fd:
content = fd.read()
self.assertNotIn("ahoj", content)
self.assertIn("cao", content)
self.create_temp_file()
self.assertRaises(Exception,
self.create_file_content,
"third",
target_file=self.temp_file)
def test_current_version_after_load(self):
PersistentObjectStorage().dump()
PersistentObjectStorage().load()
self.assertEqual(VERSION_REQURE_FILE,
PersistentObjectStorage().storage_file_version)
self.assertNotEqual({}, PersistentObjectStorage().metadata)
self.assertIn("version_storage_file",
PersistentObjectStorage().metadata)
def setUp(self) -> None:
PersistentObjectStorage().mode = StorageMode.default
super().setUp()
self.file_name = None
self.temp_dir = None
self.temp_file = None
self.response_dir = tempfile.mkdtemp(prefix="data_store")
self.response_file = os.path.join(self.response_dir, "storage_test.yaml")
PersistentObjectStorage().storage_file = self.response_file
PersistentObjectStorage().dump_after_store = True
def testDictWithList(self):
DataMiner().data_type = DataTypes.DictWithList
self.store_key()
print(PersistentObjectStorage().storage_object)
processor = DictProcessing(PersistentObjectStorage().storage_object)
processor.simplify()
self.assertIn(
"'a': {'d': {'e': {'%s': [" % DataMiner().key,
str(PersistentObjectStorage().storage_object),
)
def testValue(self):
DataMiner().data_type = DataTypes.Value
self.store_key()
print(PersistentObjectStorage().storage_object)
processor = DictProcessing(PersistentObjectStorage().storage_object)
processor.simplify()
self.assertIn(
"'a': {'d': {'e': {'metadata'",
str(PersistentObjectStorage().storage_object),
)
def apply_fn():
"""
This function is used when installed as sitecustomize.py script
to enable replacing system, please set env vars RESPONSE_FILE
REPLACEMENT_FILE, see doc string of this file
"""
# file name of storage file
storage_file = os.getenv(ENV_STORAGE_FILE)
# file name of replaces for updated import system
replacement_file = os.getenv(ENV_REPLACEMENT_FILE)
if_latency = os.getenv(ENV_APPLY_LATENCY)
replacement_var = os.getenv(ENV_REPLACEMENT_NAME, REPLACE_DEFAULT_KEY)
debug_print(
f"You have patched version of your python by requre project "
f"(python {sys.version_info.major}.{sys.version_info.minor}, {__file__}) "
)
if not (storage_file and replacement_file):
debug_print(f"\tYou have to set {ENV_STORAGE_FILE} and "
f"{ENV_REPLACEMENT_FILE} env variables to work properly")
else:
if not os.path.exists(replacement_file):
raise FileExistsError(
f"{replacement_file} has to exist to work properly "
f"(python file with replacements definition)")
if if_latency:
debug_print("Use latency for function calls")
DataMiner().use_latency = True
PersistentObjectStorage().storage_file = storage_file
spec = importlib.util.spec_from_file_location("replacements",
replacement_file)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if hasattr(module, replacement_var):
replacement = getattr(module, replacement_var)
debug_print(f"Replaces: {replacement}")
if isinstance(replacement, UpgradeImportSystem):
debug_print(
f"{replacement_var} is {UpgradeImportSystem.__name__} object"
)
elif isinstance(replacement, list):
debug_print(
f"{replacement_var} is list of replacements, apply upgrading"
)
upgrade_import_system(filters=replacement)
else:
raise_error(
126, f"Bad type of {replacement_var}, see documentation")
else:
raise_error(
125,
f"in {replacement_file} there is not defined '{replacement_var}' variable",
)
# register dump command, when python finish
atexit.register(PersistentObjectStorage().dump)
def test_no_version(self):
"""
Check if storage is able to read and write version to persistent storage file
"""
self.assertEqual({}, PersistentObjectStorage().metadata)
self.assertEqual(0, PersistentObjectStorage().storage_file_version)
self.assertEqual(
None,
PersistentObjectStorage().metadata.get(
PersistentObjectStorage().version_key),
)
self.assertFalse(os.path.exists(self.response_file))
def setUp(self) -> None:
super().setUp()
self.file_name = None
self.temp_dir = None
self.temp_file = None
self.response_dir = tempfile.mkdtemp(prefix="data_store")
self.response_file = os.path.join(self.response_dir,
"storage_test.yaml")
PersistentObjectStorage().storage_file = self.response_file
PersistentObjectStorage().dump_after_store = True
PersistentObjectStorage()._is_write_mode = True
StoreFiles.counter = 0
def testFunctionDecoratorNotFound(self):
"""
Check if it fails with Exception in case request is not stored
"""
self.requests.post = RequestResponseHandling.decorator_all_keys(
self.requests.post)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
self.assertRaises(Exception,
self.requests.post,
self.domain,
data={"a": "b"})
def testFunctionDecorator(self):
"""
Test main purpose of the class, decorate class and use it then
"""
decorated_own = ObjectStorage.decorator_all_keys(OwnClass)
obj_before = decorated_own(1)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
obj_after = decorated_own(1)
self.assertEqual(obj_before.num, obj_after.num)
# all objects are already read, next have to fail
self.assertRaises(Exception, decorated_own, 1)
def _copy_logic(cls, pers_storage: PersistentObjectStorage, pathname: str,
keys: list) -> None:
"""
Internal function. Copy files to or back from persisten storage
It will create tar archive with tar_compression and stores it to Persistent Storage
"""
logger.debug(f"Copy files {pathname} -> {keys}")
logger.debug(f"Persistent Storage mode: {pers_storage.mode}")
original_cwd = os.getcwd()
if pers_storage.do_store(keys=cls.basic_ps_keys + keys):
try:
artifact_name = os.path.basename(pathname)
artifact_path = os.path.dirname(pathname)
os.chdir(artifact_path)
with BytesIO() as fileobj:
with tarfile.open(mode=f"x:{cls.tar_compression}",
fileobj=fileobj) as tar_store:
tar_store.add(name=artifact_name)
fileobj.seek(0)
metadata = {DataMiner().LATENCY_KEY: 0}
pers_storage.store(
keys=cls.basic_ps_keys + keys,
values=fileobj.read(),
metadata=metadata,
)
finally:
os.chdir(original_cwd)
else:
value = pers_storage[cls.basic_ps_keys + keys]
with BytesIO(value) as fileobj:
with tarfile.open(mode=f"r:{cls.tar_compression}",
fileobj=fileobj) as tar_store:
tarinfo_1st_member = tar_store.getmembers()[0]
if tarinfo_1st_member.isfile():
with open(pathname, mode="wb") as output_file:
output_file.write(
tar_store.extractfile(
tarinfo_1st_member).read())
else:
for tar_item in tar_store.getmembers():
# we have to modify path of files to remove topdir
if len(tar_item.name.split(os.path.sep, 1)) > 1:
tar_item.name = tar_item.name.split(
os.path.sep, 1)[1]
else:
tar_item.name = "."
try:
tar_store.extract(tar_item, path=pathname)
except IOError:
# rewrite readonly files if necessary
os.remove(os.path.join(pathname,
tar_item.name))
tar_store.extract(tar_item, path=pathname)
def testFunctionDecorator(self):
"""
Test main purpose of the class, decorate post function and use it then
"""
self.requests.post = RequestResponseHandling.decorator_all_keys(
self.requests.post)
response_before = self.requests.post(self.domain)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
response_after = self.requests.post(self.domain)
self.assertEqual(response_before.text, response_after.text)
self.assertRaises(Exception, self.requests.post, self.domain)
def testExecuteWrapper(self):
"""
test if it is able to use explicit decorator_all_keys for storing object handling
:return:
"""
obj_before = ObjectStorage.execute_all_keys(OwnClass, 1)
PersistentObjectStorage().dump()
PersistentObjectStorage()._is_write_mode = False
obj_after = ObjectStorage.execute_all_keys(OwnClass, 1)
self.assertEqual(obj_before.num, obj_after.num)
# all objects are already read, next have to fail
self.assertRaises(Exception, ObjectStorage.execute_all_keys, OwnClass,
1)
def testCallNoDebug(self):
DataMiner().store_arg_debug_metadata = False
OwnClass.get_num = ObjectStorage.decorator_plain(OwnClass.get_num)
test_obj = OwnClass(3)
obj_before_4 = test_obj.get_num(4)
PersistentObjectStorage().dump()
PersistentObjectStorage().mode = StorageMode.read
obj_after_4 = test_obj.get_num(4)
obj_after_4_meta = DataMiner().metadata
self.assertEqual(obj_before_4, obj_after_4)
self.assertIsInstance(obj_after_4, int)
self.assertEqual(
obj_after_4_meta.get(DataMiner().METADATA_ARG_DEBUG_KEY), None)
def setUp(self):
self.token = os.environ.get("GITHUB_TOKEN")
test_name = self.id() or "all"
persistent_data_file = os.path.join(
PERSISTENT_DATA_PREFIX, f"test_github_data_{test_name}.yaml")
PersistentObjectStorage().storage_file = persistent_data_file
if PersistentObjectStorage(
).mode == StorageMode.write and not self.token:
raise EnvironmentError("please set GITHUB_TOKEN env variables")
self.service = GithubService(token=self.token, read_only=True)
self.ogr_project = self.service.get_project(namespace="packit-service",
repo="ogr")
