def test_create_file_content_positional(self): """ Similar to test_create_file_content, but test it also via positional parameters and mixing them """ self.create_temp_file() self.create_file_content("ahoj", self.temp_file) self.create_file_content("cao", self.temp_file) PersistentObjectStorage().cassette.dump() PersistentObjectStorage().cassette.mode = StorageMode.read self.create_temp_file() self.create_file_content("first", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option self.create_temp_file() self.create_file_content("second", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.create_temp_file() self.assertRaises(Exception, self.create_file_content, "third", target_file=self.temp_file)
def test(self): """ Mixing command wrapper with file storage """ self.assertEqual(StoreFiles.counter, 0) self.create_temp_file() self.create_file_content("ahoj", target_file=self.temp_file) self.assertEqual(StoreFiles.counter, 1) self.create_file_content("cao", target_file=self.temp_file) self.assertEqual(StoreFiles.counter, 2) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 before = str(PersistentObjectStorage().storage_object) self.create_file_content("ahoj", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) self.create_file_content("cao", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) after = str(PersistentObjectStorage().storage_object) self.assertGreater(len(before), len(after)) self.assertIn("True", before)
def test_create_dir_content(self): """ Check if properly store and restore directory content """ self.create_temp_dir() self.create_dir_content(filename="ahoj", target_dir=self.temp_dir, content="ciao") self.assertIn("ahoj", os.listdir(self.temp_dir)) with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd: content = fd.read() self.assertIn("ciao", content) PersistentObjectStorage().cassette.dump() PersistentObjectStorage().cassette.mode = StorageMode.read self.create_temp_dir() self.create_dir_content(filename="nonsense", target_dir=self.temp_dir, content="bad") self.assertIn("ahoj", os.listdir(self.temp_dir)) self.assertNotIn("nonsense", os.listdir(self.temp_dir)) with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd: content = fd.read() self.assertIn("ciao", content) self.assertNotIn("bad", content)
def test_create_file_content(self): """ test if is able store files via decorated function create_file_content """ self.assertEqual(StoreFiles.counter, 0) self.create_temp_file() self.assertEqual( "value", self.create_file_content("ahoj", target_file=self.temp_file)) self.assertEqual(StoreFiles.counter, 1) self.create_file_content("cao", target_file=self.temp_file) self.assertEqual(StoreFiles.counter, 2) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 self.create_file_content("first", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) self.create_file_content("second", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.assertRaises(Exception, self.create_file_content, "third", target_file=self.temp_file)
def test_create_file_content_positional(self): """ Similar to test_create_file_content, but test it also via positional parameters and mixing them """ self.assertEqual(StoreFiles.counter, 0) self.create_temp_file() self.create_file_content("ahoj", self.temp_file) self.assertEqual(StoreFiles.counter, 1) self.create_file_content("cao", self.temp_file) self.assertEqual(StoreFiles.counter, 2) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 self.create_temp_file() self.create_file_content("first", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option self.create_temp_file() self.create_file_content("second", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.create_temp_file() self.assertRaises(Exception, self.create_file_content, "third", target_file=self.temp_file)
def test_create_file_content(self): """ test if is able store files via decorated function create_file_content """ self.create_temp_file() self.assertEqual( "value", self.create_file_content("ahoj", target_file=self.temp_file)) self.create_file_content("cao", target_file=self.temp_file) PersistentObjectStorage().cassette.dump() PersistentObjectStorage().cassette.mode = StorageMode.read self.create_file_content("first", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) self.create_file_content("second", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.assertRaises(Exception, self.create_file_content, "third", target_file=self.temp_file)
def setUp(self): self.github_token = os.environ.get("GITHUB_TOKEN") self.pagure_token = os.environ.get("PAGURE_TOKEN") self.gitlab_token = os.environ.get("GITLAB_TOKEN") or "some_token" test_name = self.id() or "all" persistent_data_file = os.path.join( PERSISTENT_DATA_PREFIX, f"test_factory_data_{test_name}.yaml") PersistentObjectStorage().storage_file = persistent_data_file if PersistentObjectStorage().is_write_mode and not self.github_token: raise EnvironmentError("please set GITHUB_TOKEN env variables") if PersistentObjectStorage().is_write_mode and not self.pagure_token: raise EnvironmentError("please set PAGURE_TOKEN env variables") if PersistentObjectStorage( ).is_write_mode and not os.environ.get("GITLAB_TOKEN"): raise EnvironmentError("please set GITLAB_TOKEN env variables") self.github_service = GithubService(token=self.github_token) self.pagure_service = PagureService(token=self.pagure_token) self.gitlab_service = GitlabService(token=self.gitlab_token, instance_url="https://gitlab.com") self.custom_instances = [ self.github_service, self.pagure_service, self.gitlab_service, ]
def test_run_command_output(self): """ check if wrapper returns proper string values in calls """ self.file_name = tempfile.mktemp() with open(self.file_name, "w") as fd: fd.write("ahoj\n") output = self.run_command_wrapper(cmd=["cat", self.file_name], output=True) self.assertIn("ahoj", output) PersistentObjectStorage().cassette.dump() PersistentObjectStorage().cassette.mode = StorageMode.read with open(self.file_name, "a") as fd: fd.write("cao\n") output = self.run_command_wrapper(cmd=["cat", self.file_name], output=True) self.assertIn("ahoj", output) self.assertNotIn("cao", output) PersistentObjectStorage().cassette.mode = StorageMode.write output = self.run_command_wrapper(cmd=["cat", self.file_name], output=True) self.assertIn("cao", output) PersistentObjectStorage().cassette.dump() PersistentObjectStorage().cassette.mode = StorageMode.read output = self.run_command_wrapper(cmd=["cat", self.file_name], output=True) self.assertIn("cao", output)
def test_create_dir_content(self): """ Check if properly store and restore directory content """ self.assertEqual(StoreFiles.counter, 0) self.create_temp_dir() self.create_dir_content(filename="ahoj", target_dir=self.temp_dir, content="ciao") self.assertEqual(StoreFiles.counter, 1) self.assertIn("ahoj", os.listdir(self.temp_dir)) with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd: content = fd.read() self.assertIn("ciao", content) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 self.create_temp_dir() self.create_dir_content(filename="nonsense", target_dir=self.temp_dir, content="bad") self.assertIn("ahoj", os.listdir(self.temp_dir)) self.assertNotIn("nonsense", os.listdir(self.temp_dir)) with open(os.path.join(self.temp_dir, "ahoj"), "r") as fd: content = fd.read() self.assertIn("ciao", content) self.assertNotIn("bad", content)
def testChangeFile(self): PersistentObjectStorage().storage_file += ".x" output = TempFile.mktemp() self.assertIn( f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_1", output, ) output = TempFile.mktemp() self.assertIn( f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_2", output, ) PersistentObjectStorage().storage_file += ".y" self.assertEqual(TempFile.counter, 2) output = TempFile.mktemp() self.assertEqual(TempFile.counter, 1) self.assertIn( f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_1", output, ) output = TempFile.mktemp() self.assertIn( f"/tmp/{os.path.basename(PersistentObjectStorage().storage_file)}/static_tmp_2", output, )
def test_create_file(self): """ Test File storage if file name is return value of function """ ofile1 = self.create_file("ahoj") ofile2 = self.create_file("cao") PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read oofile1 = self.create_file("first") with open(ofile1, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) with open(oofile1, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option oofile2 = self.create_file("second") with open(oofile2, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.assertEqual(ofile2, oofile2)
def setUp(self): super().setUp() print(PersistentObjectStorage().storage_file) self.github_token = os.environ.get("GITHUB_TOKEN") self.pagure_token = os.environ.get("PAGURE_TOKEN") self.gitlab_token = os.environ.get("GITLAB_TOKEN") or "some_token" if PersistentObjectStorage().mode == StorageMode.write: if not self.github_token: raise EnvironmentError( "You are in requre write mode, please set GITHUB_TOKEN env variables" ) if not self.pagure_token: raise EnvironmentError( "You are in requre write mode, please set PAGURE_TOKEN env variables" ) if not os.environ.get("GITLAB_TOKEN"): raise EnvironmentError( "You are in requre write mode, please set GITLAB_TOKEN env variables" ) self.github_service = GithubService(token=self.github_token) self.pagure_service = PagureService(token=self.pagure_token) self.gitlab_service = GitlabService( token=self.gitlab_token, instance_url="https://gitlab.com" ) self.custom_instances = [ self.github_service, self.pagure_service, self.gitlab_service, ]
def test_create_file(self): """ File storage where it try to guess what to store, based on *args and **kwargs values """ self.create_temp_file() self.write_to_file("ahoj", self.temp_file) self.write_to_file("cao", self.temp_file) PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read self.create_temp_file() self.write_to_file("ahoj", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option self.create_temp_file() self.write_to_file("cao", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.create_temp_file() self.assertRaises( Exception, self.create_file_content, "third", target_file=self.temp_file )
def test(self): """ Mixing command wrapper with file storage """ self.create_temp_file() self.create_file_content("ahoj", target_file=self.temp_file) self.create_file_content("cao", target_file=self.temp_file) PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read before = str(PersistentObjectStorage().storage_object) self.create_file_content("ahoj", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) self.create_file_content("cao", target_file=self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) after = str(PersistentObjectStorage().storage_object) self.assertGreater(len(before), len(after)) self.assertIn("True", before)
def test_create_file(self): """ Test File storage if file name is return value of function """ self.assertEqual(StoreFiles.counter, 0) ofile1 = self.create_file("ahoj") self.assertEqual(StoreFiles.counter, 1) ofile2 = self.create_file("cao") self.assertEqual(StoreFiles.counter, 2) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 oofile1 = self.create_file("first") with open(ofile1, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) with open(oofile1, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option oofile2 = self.create_file("second") with open(oofile2, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.assertEqual(ofile2, oofile2)
def test_create_file(self): """ File storage where it try to guess what to store, based on *args and **kwargs values """ self.assertEqual(StoreFiles.counter, 0) self.create_temp_file() self.create_file("ahoj", self.temp_file) self.assertEqual(StoreFiles.counter, 1) self.create_file("cao", self.temp_file) self.assertEqual(StoreFiles.counter, 2) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False StoreFiles.counter = 0 self.create_temp_file() self.create_file("first", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertIn("ahoj", content) self.assertNotIn("cao", content) # mix with positional option self.create_temp_file() self.create_file("second", self.temp_file) with open(self.temp_file, "r") as fd: content = fd.read() self.assertNotIn("ahoj", content) self.assertIn("cao", content) self.create_temp_file() self.assertRaises(Exception, self.create_file_content, "third", target_file=self.temp_file)
def test_current_version_after_load(self): PersistentObjectStorage().dump() PersistentObjectStorage().load() self.assertEqual(VERSION_REQURE_FILE, PersistentObjectStorage().storage_file_version) self.assertNotEqual({}, PersistentObjectStorage().metadata) self.assertIn("version_storage_file", PersistentObjectStorage().metadata)
def setUp(self) -> None: PersistentObjectStorage().mode = StorageMode.default super().setUp() self.file_name = None self.temp_dir = None self.temp_file = None self.response_dir = tempfile.mkdtemp(prefix="data_store") self.response_file = os.path.join(self.response_dir, "storage_test.yaml") PersistentObjectStorage().storage_file = self.response_file PersistentObjectStorage().dump_after_store = True
def testDictWithList(self): DataMiner().data_type = DataTypes.DictWithList self.store_key() print(PersistentObjectStorage().storage_object) processor = DictProcessing(PersistentObjectStorage().storage_object) processor.simplify() self.assertIn( "'a': {'d': {'e': {'%s': [" % DataMiner().key, str(PersistentObjectStorage().storage_object), )
def testValue(self): DataMiner().data_type = DataTypes.Value self.store_key() print(PersistentObjectStorage().storage_object) processor = DictProcessing(PersistentObjectStorage().storage_object) processor.simplify() self.assertIn( "'a': {'d': {'e': {'metadata'", str(PersistentObjectStorage().storage_object), )
def apply_fn(): """ This function is used when installed as sitecustomize.py script to enable replacing system, please set env vars RESPONSE_FILE REPLACEMENT_FILE, see doc string of this file """ # file name of storage file storage_file = os.getenv(ENV_STORAGE_FILE) # file name of replaces for updated import system replacement_file = os.getenv(ENV_REPLACEMENT_FILE) if_latency = os.getenv(ENV_APPLY_LATENCY) replacement_var = os.getenv(ENV_REPLACEMENT_NAME, REPLACE_DEFAULT_KEY) debug_print( f"You have patched version of your python by requre project " f"(python {sys.version_info.major}.{sys.version_info.minor}, {__file__}) " ) if not (storage_file and replacement_file): debug_print(f"\tYou have to set {ENV_STORAGE_FILE} and " f"{ENV_REPLACEMENT_FILE} env variables to work properly") else: if not os.path.exists(replacement_file): raise FileExistsError( f"{replacement_file} has to exist to work properly " f"(python file with replacements definition)") if if_latency: debug_print("Use latency for function calls") DataMiner().use_latency = True PersistentObjectStorage().storage_file = storage_file spec = importlib.util.spec_from_file_location("replacements", replacement_file) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) if hasattr(module, replacement_var): replacement = getattr(module, replacement_var) debug_print(f"Replaces: {replacement}") if isinstance(replacement, UpgradeImportSystem): debug_print( f"{replacement_var} is {UpgradeImportSystem.__name__} object" ) elif isinstance(replacement, list): debug_print( f"{replacement_var} is list of replacements, apply upgrading" ) upgrade_import_system(filters=replacement) else: raise_error( 126, f"Bad type of {replacement_var}, see documentation") else: raise_error( 125, f"in {replacement_file} there is not defined '{replacement_var}' variable", ) # register dump command, when python finish atexit.register(PersistentObjectStorage().dump)
def test_no_version(self): """ Check if storage is able to read and write version to persistent storage file """ self.assertEqual({}, PersistentObjectStorage().metadata) self.assertEqual(0, PersistentObjectStorage().storage_file_version) self.assertEqual( None, PersistentObjectStorage().metadata.get( PersistentObjectStorage().version_key), ) self.assertFalse(os.path.exists(self.response_file))
def setUp(self) -> None: super().setUp() self.file_name = None self.temp_dir = None self.temp_file = None self.response_dir = tempfile.mkdtemp(prefix="data_store") self.response_file = os.path.join(self.response_dir, "storage_test.yaml") PersistentObjectStorage().storage_file = self.response_file PersistentObjectStorage().dump_after_store = True PersistentObjectStorage()._is_write_mode = True StoreFiles.counter = 0
def testFunctionDecoratorNotFound(self): """ Check if it fails with Exception in case request is not stored """ self.requests.post = RequestResponseHandling.decorator_all_keys( self.requests.post) PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read self.assertRaises(Exception, self.requests.post, self.domain, data={"a": "b"})
def testFunctionDecorator(self): """ Test main purpose of the class, decorate class and use it then """ decorated_own = ObjectStorage.decorator_all_keys(OwnClass) obj_before = decorated_own(1) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False obj_after = decorated_own(1) self.assertEqual(obj_before.num, obj_after.num) # all objects are already read, next have to fail self.assertRaises(Exception, decorated_own, 1)
def _copy_logic(cls, pers_storage: PersistentObjectStorage, pathname: str, keys: list) -> None: """ Internal function. Copy files to or back from persisten storage It will create tar archive with tar_compression and stores it to Persistent Storage """ logger.debug(f"Copy files {pathname} -> {keys}") logger.debug(f"Persistent Storage mode: {pers_storage.mode}") original_cwd = os.getcwd() if pers_storage.do_store(keys=cls.basic_ps_keys + keys): try: artifact_name = os.path.basename(pathname) artifact_path = os.path.dirname(pathname) os.chdir(artifact_path) with BytesIO() as fileobj: with tarfile.open(mode=f"x:{cls.tar_compression}", fileobj=fileobj) as tar_store: tar_store.add(name=artifact_name) fileobj.seek(0) metadata = {DataMiner().LATENCY_KEY: 0} pers_storage.store( keys=cls.basic_ps_keys + keys, values=fileobj.read(), metadata=metadata, ) finally: os.chdir(original_cwd) else: value = pers_storage[cls.basic_ps_keys + keys] with BytesIO(value) as fileobj: with tarfile.open(mode=f"r:{cls.tar_compression}", fileobj=fileobj) as tar_store: tarinfo_1st_member = tar_store.getmembers()[0] if tarinfo_1st_member.isfile(): with open(pathname, mode="wb") as output_file: output_file.write( tar_store.extractfile( tarinfo_1st_member).read()) else: for tar_item in tar_store.getmembers(): # we have to modify path of files to remove topdir if len(tar_item.name.split(os.path.sep, 1)) > 1: tar_item.name = tar_item.name.split( os.path.sep, 1)[1] else: tar_item.name = "." try: tar_store.extract(tar_item, path=pathname) except IOError: # rewrite readonly files if necessary os.remove(os.path.join(pathname, tar_item.name)) tar_store.extract(tar_item, path=pathname)
def testFunctionDecorator(self): """ Test main purpose of the class, decorate post function and use it then """ self.requests.post = RequestResponseHandling.decorator_all_keys( self.requests.post) response_before = self.requests.post(self.domain) PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read response_after = self.requests.post(self.domain) self.assertEqual(response_before.text, response_after.text) self.assertRaises(Exception, self.requests.post, self.domain)
def testExecuteWrapper(self): """ test if it is able to use explicit decorator_all_keys for storing object handling :return: """ obj_before = ObjectStorage.execute_all_keys(OwnClass, 1) PersistentObjectStorage().dump() PersistentObjectStorage()._is_write_mode = False obj_after = ObjectStorage.execute_all_keys(OwnClass, 1) self.assertEqual(obj_before.num, obj_after.num) # all objects are already read, next have to fail self.assertRaises(Exception, ObjectStorage.execute_all_keys, OwnClass, 1)
def testCallNoDebug(self): DataMiner().store_arg_debug_metadata = False OwnClass.get_num = ObjectStorage.decorator_plain(OwnClass.get_num) test_obj = OwnClass(3) obj_before_4 = test_obj.get_num(4) PersistentObjectStorage().dump() PersistentObjectStorage().mode = StorageMode.read obj_after_4 = test_obj.get_num(4) obj_after_4_meta = DataMiner().metadata self.assertEqual(obj_before_4, obj_after_4) self.assertIsInstance(obj_after_4, int) self.assertEqual( obj_after_4_meta.get(DataMiner().METADATA_ARG_DEBUG_KEY), None)
def setUp(self): self.token = os.environ.get("GITHUB_TOKEN") test_name = self.id() or "all" persistent_data_file = os.path.join( PERSISTENT_DATA_PREFIX, f"test_github_data_{test_name}.yaml") PersistentObjectStorage().storage_file = persistent_data_file if PersistentObjectStorage( ).mode == StorageMode.write and not self.token: raise EnvironmentError("please set GITHUB_TOKEN env variables") self.service = GithubService(token=self.token, read_only=True) self.ogr_project = self.service.get_project(namespace="packit-service", repo="ogr")