class Test_FileUtil(TestCase): path_no_drive = 'temp' fn = 'test.csv' yaml = 'example.yaml' text_fn = 'test.txt' def __init__(self, *args, **kwargs): super(Test_FileUtil, self).__init__(*args, **kwargs) self.path = r'c:\temp' if platform.system() == 'Windows' else r'/tmp' self._fu = FileUtil() self._du = DateUtil() self.features_dict = { 'book': "Hitchhiker's Guide", 'characters': { 'answer': 42, 'name': 'Dent. Arthur Dent.' } } @classmethod def tearDownClass(cls) -> None: fu = FileUtil() path = r'c:\temp' if platform.system() == 'Windows' else r'/tmp' fu.delete_file(fu.qualified_path(path, cls.yaml)) fu.delete_file(fu.qualified_path(path, cls.fn)) fu.delete_file(fu.qualified_path(path, cls.text_fn)) @property def path(self): return self._path @path.setter def path(self, p): self._path = p def create_csv(self): lines = [ ',col1,col2', '0,1,3', '1,2,4', ] filename = self._fu.qualified_path(self.path, self.fn) self._fu.write_text_file(filename, lines) logger.debug(f'create_csv to {self.path}{sep}{self.fn}.') def create_yaml(self, keys: list, vals: list): writeMe = [] for i in range(len(keys)): writeMe.append(f'{keys[i]}: {vals[i]}') qualifiedPath = self._fu.qualified_path(self.path, self.yaml) self._fu.write_text_file(filename=qualifiedPath, lines=writeMe) def generate_text_lines(self, how_many: int = 10, width: int = None) -> List[str]: if width: ans = [ '{0:*^{width}}'.format(i, width=width) for i in range(how_many) ] return ans return [f'Line {i}' for i in range(how_many)] def create_text_file(self, filename: str, how_many: int = 10, width: int = None): lines = self.generate_text_lines(how_many, width) self._fu.write_text_file(filename, lines) @logit() def test_is_windows(self): with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Linux' mocked_fu = FileUtil() test1 = mocked_fu.is_Windows self.assertFalse(test1) with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Windows' mocked_fu = FileUtil() self.assertTrue(mocked_fu.is_Windows) @logit() def test_dump_yaml(self): yaml_fn = self._fu.qualified_path(self.path, self.yaml) self._fu.dump_yaml(yaml_fn, self.features_dict) self.assertTrue(self._fu.file_exists(yaml_fn)) actual = self._fu.read_yaml(yaml_fn) self.assertDictEqual(self.features_dict, actual) @logit() def test_current_directory(self): logger.debug( f'current working dir is really {self._fu.current_directory()}') my_mock_dir = r'\synthesys\testing' with mock.patch('FileUtil.getcwd', return_value=my_mock_dir): actual = self._fu.current_directory() self.assertEqual(actual, my_mock_dir) def test_read_text_file(self): filename = self._fu.qualified_path(self.path, self.text_fn) how_many_lines = randrange(10) + 2 self.create_text_file(filename, how_many_lines) expected = self.generate_text_lines(how_many_lines) actual = [x.rstrip() for x in self._fu.read_text_file(filename) ] # must remove newline chars self.assertListEqual(expected, actual) @logit() def test_read_text_file_err(self): # test an IO error filename = self._fu.qualified_path(self.path, self.text_fn) with mock.patch('FileUtil.open', create=True) as mocked_open: mocked_open.side_effect = IOError() self._fu.read_text_file(filename) @logit() def test_read_yaml(self): keys = ['firstname', 'lastname', 'zip'] vals = ['Rajah', 'Chacko', 28269] self.create_yaml(keys, vals) qualifiedPath = self._fu.qualified_path(self.path, self.yaml) d = self._fu.read_yaml(yamlFile=qualifiedPath) logger.debug(f'Contents of yaml: {d}') self.assertEqual(list(d.keys()), keys) self.assertEqual(vals[0], d[keys[0]]) @logit() @mock.patch('FileUtil.safe_load') def test_read_yaml_err(self, mock_obj): yaml_fn = self._fu.qualified_path(self.path, self.yaml) self.create_text_file(yaml_fn) mock_obj.side_effect = YAMLError('mock error') actual = self._fu.read_yaml(yamlFile=yaml_fn) self.assertIsNone(actual) @logit() def test_qualified_path(self): # Test 1. Normal case. expected = self.path + sep + self.fn actual = self._fu.qualified_path(self.path, self.fn) self.assertEqual(actual, expected, "Test 1 fail") # Test 2. Using an array and a Linux mock. with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Windows' mocked_fu = FileUtil() dir_to_path = mocked_fu.separator.join( ['C:', 'dir', 'to', 'path']) # should be C:\dir\to\path for Windows pathArray = dir_to_path.split(mocked_fu.separator) expected = dir_to_path + mocked_fu.separator + self.fn self.assertEqual( expected, mocked_fu.fully_qualified_path(pathArray, self.fn, dir_path_is_array=True), "Test 2 fail") # Test 3, using a windows path with a drive exp3 = r'c:\temp\subdir\subsubdir' exp3_array = exp3.split(_BACKSLASH) test3_with_fn = deepcopy(exp3_array) test3_with_fn.append(self.fn) test3 = _BACKSLASH.join(test3_with_fn) with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Windows' mocked_fu = FileUtil() actual = mocked_fu.qualified_path(dirPath=exp3_array, filename=self.fn, dir_path_is_array=True) self.assertEqual(test3, actual, "Test 3 fail") @logit() def test_fully_qualified_path(self): # Test 1, Windows (should be unchanged) path1 = r'c:\temp\subdir\subsubdir' with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Windows' mocked_fu = FileUtil() exp1 = path1 + mocked_fu.separator + self.fn self.assertEqual( exp1, mocked_fu.fully_qualified_path(dirPath=path1, filename=self.fn), 'Test 1 fail') # Test 2, Linux without the leading / test2 = r'dir/to/path' # Test 3, Linux with the leading / (should be unchanged) with mock.patch('platform.system') as mocked_platform: mocked_platform.return_value = 'Linux' mocked_fu = FileUtil() exp2 = mocked_fu.separator + test2 + mocked_fu.separator + self.fn self.assertEqual( exp2, mocked_fu.fully_qualified_path(dirPath=test2, filename=self.fn, dir_path_is_array=False), "Test 2 fail") test3 = mocked_fu.separator + test2 exp3 = test3 + mocked_fu.separator + self.fn self.assertEqual( exp3, mocked_fu.fully_qualified_path(dirPath=test3, filename=self.fn, dir_path_is_array=False), "Test 3 fail") @logit() def test_split_qualified_path(self): fn = 'test.txt' qpath = self._fu.qualified_path(self.path, fn) # Test 1. c:\temp for Windows or /tmp for Linux. which_test = 1 splitpath, splitfn = self._fu.split_qualified_path(qpath, makeArray=False) self.assertEqual(splitpath, self.path, f'Test {which_test}. Paths should be equal.') self.assertEqual(splitfn, fn, f'Test {which_test}. File names should be equal.') # Test 2. Split paths into arrays. which_test = 2 pathArray, splitfn = self._fu.split_qualified_path(qpath, makeArray=True) expected = self.path.split(sep) self.assertEqual(pathArray, expected, f'Test {which_test}. Paths should be equal.') self.assertEqual(splitfn, fn, f'Test {which_test}. File names should be equal.') # Test 3. Try a more complex path. which_test = 3 complex_path = r'C:\Users\Owners\Documents\Tickers.csv' if platform.system( ) == 'Windows' else r'/tmp/parent/child/Tickers.csv' pathArray, splitfn = self._fu.split_qualified_path(complex_path, makeArray=True) expected = complex_path.split(sep) expected.pop() # Pop off the last el, which is the file name. self.assertEqual(pathArray, expected, f'Test {which_test}. Paths should be equal.') self.assertEqual(splitfn, 'Tickers.csv', f'Test {which_test}. File names should be equal.') @logit() def test_split_file_name(self): expected_file = "file" expected_ext = ".ext" expected_fn = expected_file + expected_ext # First test with just file.ext actual_file, actual_ext = self._fu.split_file_name(expected_fn) self.assertEqual(actual_file, expected_file) self.assertEqual(actual_ext, expected_ext) # Another test with path/file.ext qpath = self._fu.qualified_path(self.path, expected_fn) actual_file, actual_ext = self._fu.split_file_name(qpath) self.assertEqual(actual_file, expected_file) self.assertEqual(actual_ext, expected_ext) @logit() def test_file_exists(self): self.create_csv() qualifiedPath = self._fu.qualified_path(self.path, self.fn) self.assertTrue(self._fu.file_exists(qualifiedPath)) qualifiedPath = self._fu.qualified_path(self.path, 'noSuchFile.xxd') self.assertFalse(self._fu.file_exists(qualifiedPath)) @logit() def test_ensure_dir(self): self._fu.ensure_dir(self.path) self.assertTrue(self._fu.dir_exists(self.path)) @logit() def test_delete_file(self): self.create_csv() qualifiedPath = self._fu.qualified_path(self.path, self.fn) # delete_file should return True the first time self.assertTrue(self._fu.delete_file(qualifiedPath)) # but return false the second time. self.assertFalse(self._fu.delete_file(qualifiedPath)) @logit() @mock.patch('FileUtil.remove') def test_delete_file_err(self, mock_obj): self.create_csv() expected_log_message = 'delete_file mocktest' mock_obj.side_effect = OSError(expected_log_message) qualifiedPath = self._fu.qualified_path(self.path, self.fn) with self.assertLogs(FileUtil.__name__, level='DEBUG') as cm: ans = self._fu.delete_file(qualifiedPath) self.assertFalse(ans) self.assertTrue( next((True for line in cm.output if expected_log_message in line), False)) @logit() def test_copy_file(self): self.create_csv() copy_fn = self.fn + '.copy' copied_file = self._fu.qualified_path(self.path, copy_fn) source_path = self._fu.qualified_path(self.path, self.fn) self._fu.copy_file(source_path, copied_file) self.assertTrue(self._fu.file_exists(source_path)) self.assertTrue(self._fu.file_exists(copied_file)) self._fu.delete_file(copied_file) @logit() @mock.patch('FileUtil.copy2') def test_copy_file_err(self, mock_obj): tmp_path = self._fu.qualified_path(self.path, 'tmp') qualifiedPath = self._fu.qualified_path(self.path, self.fn) expected_log_message = 'copy_file mocktest' mock_obj.side_effect = IOError(expected_log_message) with self.assertLogs(FileUtil.__name__, level='DEBUG') as cm: _ = self._fu.copy_file(qualifiedPath, tmp_path) self.assertTrue( next((True for line in cm.output if expected_log_message in line), False)) @logit() def test_getList(self): dir_name = r'c:\temp' flist = self._fu.getList(dir_name) logger.debug(f'All list is: {flist}') def isFile_side_effect(*args, **kwargs) -> bool: """ Side effect for mocking test_get_files. Returns True if there is a .txt in the filename. Not great, but ok for mocking. :param args: :param kwargs: :return: """ return mock_is_file(args[1]) def isDir_side_effect(*args) -> bool: return mock_is_dir(args[1]) @logit() @mock.patch('FileUtil.isfile') @mock.patch('FileUtil.listdir') def test_get_files(self, mock_listdir, mock_isfile): dir_name = r'\nosuchdir' file_list = ['filea.txt', 'fileb.txt', 'filec.txt', 'somedir'] mock_listdir.return_value = file_list mock_isfile.side_effect = self.isFile_side_effect actual = self._fu.get_files(dir_name) expected = [f for f in file_list if mock_is_file(f) ] # Condition must match isFile_side_effect self.assertListEqual(expected, actual) @logit() @mock.patch('FileUtil.isdir') @mock.patch('FileUtil.listdir') def test_get_dirs(self, mock_listdir, mock_isdir): dir_name = r'\nosuchdir' file_list = ['filea.txt', 'fileb.txt', 'filec.txt', 'somedir'] mock_listdir.return_value = file_list mock_isdir.side_effect = self.isDir_side_effect actual = self._fu.get_dirs(dir_name) expected = [f for f in file_list if mock_is_dir(f) ] # Condition must match isDir_side_effect self.assertListEqual(expected, actual) @logit() def test_getRecursiveList(self): dir_name = r'\nosuchdir' file_list = ['filea.txt', 'fileb.txt', 'filec.txt'] actual = self._fu.getRecursiveList(dir_name) self.assertListEqual(actual, []) # Since no such dir, should be empty list eu = ExecUtil() exec_file = eu.exec_file_path() dir_name, _ = self._fu.split_qualified_path(exec_file) logger.debug(f'dir name is: {dir_name}') with mock.patch('FileUtil.listdir', return_value=file_list): actual = self._fu.getRecursiveList(dir_name) expected = [ self._fu.fully_qualified_path(dirPath=dir_name, filename=f) for f in file_list ] self.assertListEqual(expected, actual) @logit() def test_load_logs_and_subdir_names(self): no_such_dir_name = r'\nosuchdir' file_list = ['filea.txt', 'fileb.csv', 'otherfile.txt'] actual = self._fu.load_logs_and_subdir_names(no_such_dir_name) self.assertListEqual(actual, []) # Since no such dir, should be empty list eu = ExecUtil() dir_name = eu.executing_directory() # ensures that dir_name is real with mock.patch('FileUtil.listdir', return_value=file_list): # Test with neither prefix nor suffix actual = self._fu.load_logs_and_subdir_names(dir_name) expected = [ self._fu.fully_qualified_path(dirPath=dir_name, filename=f) for f in file_list ] self.assertListEqual(expected, actual) # Test for suffixes ending in .txt suffix = '.txt' actual = self._fu.load_logs_and_subdir_names(dir_name, requiredSuffix=suffix) txt_only = [ self._fu.fully_qualified_path(dirPath=dir_name, filename=f) for f in file_list if f.endswith(suffix) ] self.assertListEqual(txt_only, actual) # Test for prefixes starting with 'file' prefix = 'file' actual = self._fu.load_logs_and_subdir_names(dir_name, requiredPrefix=prefix) file_only = [ self._fu.fully_qualified_path(dirPath=dir_name, filename=f) for f in file_list if f.startswith(prefix) ] self.assertListEqual(file_only, actual) @logit() @mock.patch('FileUtil.isfile') @mock.patch('FileUtil.listdir') def test_cull_existing_files(self, mock_listdir, mock_isfile): dir_name = r'\nosuchdir' file_list = ['filea.txt', 'fileb.txt', 'filec.txt', 'somedir'] mock_listdir.return_value = file_list mock_isfile.side_effect = self.isFile_side_effect qualified_file_list = [ self._fu.qualified_path(dirPath=dir_name, filename=f) for f in file_list ] actual = self._fu.cull_existing_files(qualified_file_list) expected = [f for f in qualified_file_list if mock_is_file(f) ] # Condition must match isFile_side_effect self.assertListEqual(expected, actual) @logit() def test_read_generator(self): filename = self._fu.qualified_path(self.path, self.text_fn) how_many_lines = 5 self.create_text_file(filename, how_many_lines) lines_read_in = 0 for i, line in enumerate(self._fu.read_generator(filename)): logger.debug(f'Read in line {i}, which contains <{line}>.') lines_read_in += 1 self.assertEqual(how_many_lines, lines_read_in) @logit() @mock.patch('FileUtil.open') def test_read_generator_err(self, mock_open): expected_log_message = 'mocked error' mock_open.side_effect = IOError(expected_log_message) filename = self._fu.qualified_path(self.path, self.text_fn) with self.assertLogs(FileUtil.__name__, level='DEBUG') as cm: for i, line in enumerate(self._fu.read_generator(filename)): x = line logger.debug(f'Read in line {i}, which contains <{x}>.') self.assertIsNone(x) logger.debug(f'Caught exception message: {cm.output}') self.assertTrue( next((True for line in cm.output if expected_log_message in line), False)) @logit() def test_file_modify_time(self): start_time = self._du.as_timestamp() keys = [ 'greeting', 'farewell', ] vals = [ 'Hello', 'Goodbye', ] self.create_yaml(keys, vals) qualifiedPath = self._fu.qualified_path(self.path, self.yaml) mod_time = self._fu.file_modify_time(qualifiedPath) mod_timestamp = self._du.as_timestamp(dt=mod_time) logger.debug( f'mod_time is {mod_timestamp}. start_time is {start_time}.') self.assertTrue((start_time - mod_timestamp) < .1) # asserting a difference of < 0.1 seconds. @logit() def test_file_modify_time2(self): start_time = self._du.as_timestamp() keys = [ 'greeting', 'farewell', ] vals = [ 'Hello', 'Goodbye', ] self.create_yaml(keys, vals) qualifiedPath = self._fu.qualified_path(self.path, self.yaml) mod_time = self._fu.file_modify_time2(qualifiedPath) mod_timestamp = self._du.as_timestamp(dt=mod_time) self.assertTrue((start_time - mod_timestamp) < .1) # asserting a difference of < 0.1 seconds. @logit() def test_file_size(self): filename = self._fu.qualified_path(self.path, self.text_fn) width = 20 how_many_lines = randrange(10) + 2 self.create_text_file(filename, how_many_lines, width) eol_len = 2 actual = self._fu.file_size(filename) self.assertEqual((width + eol_len) * how_many_lines, actual) @logit() def test_list_modules(self): mods = [] for mod_name in self._fu.list_module_contents(module_name='itertools'): mods.append(mod_name) self.assertTrue('__docs__' in mods) @logit() def test_list_modules(self): doc = self._fu.list_module_attributes('itertools', True) logger.debug('{}'.format(doc)) mods = [] for mod_name in self._fu.list_modules(module_name='itertools'): mods.append(mod_name) self.assertTrue('__doc__' in mods) self.assertTrue('__name__' in mods)