def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform == 'win32':
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
with self.assertRaises(OSError):
fs.maybe_make_directory(sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_sep(self):
fs = FileSystem()
self.assertEqual(fs.sep, os.sep)
self.assertEqual(fs.join('foo', 'bar'),
os.path.join('foo', 'bar'))
class TextFileReaderTest(LoggingTestCase):
class MockProcessor(ProcessorBase):
"""A processor for test purposes.
This processor simply records the parameters passed to its process()
method for later checking by the unittest test methods.
"""
def __init__(self):
self.processed = []
"""The parameters passed for all calls to the process() method."""
def should_process(self, file_path):
return not file_path.endswith('should_not_process.txt')
def process(self, lines, file_path, test_kwarg=None):
self.processed.append((lines, file_path, test_kwarg))
def setUp(self):
LoggingTestCase.setUp(self)
# FIXME: This should be a MockFileSystem once TextFileReader is moved entirely on top of FileSystem.
self.filesystem = FileSystem()
self._temp_dir = str(self.filesystem.mkdtemp())
self._processor = TextFileReaderTest.MockProcessor()
self._file_reader = TextFileReader(self.filesystem, self._processor)
def tearDown(self):
LoggingTestCase.tearDown(self)
self.filesystem.rmtree(self._temp_dir)
def _create_file(self, rel_path, text):
"""Create a file with given text and return the path to the file."""
# FIXME: There are better/more secure APIs for creating tmp file paths.
file_path = self.filesystem.join(self._temp_dir, rel_path)
self.filesystem.write_text_file(file_path, text)
return file_path
def _passed_to_processor(self):
"""Return the parameters passed to MockProcessor.process()."""
return self._processor.processed
def _assert_file_reader(self, passed_to_processor, file_count):
"""Assert the state of the file reader."""
self.assertEqual(passed_to_processor, self._passed_to_processor())
self.assertEqual(file_count, self._file_reader.file_count)
def test_process_file__does_not_exist(self):
try:
self._file_reader.process_file('does_not_exist.txt')
except SystemExit as err:
self.assertEqual(str(err), '1')
else:
self.fail('No Exception raised.')
self._assert_file_reader([], 1)
self.assertLog(["ERROR: File does not exist: 'does_not_exist.txt'\n"])
def test_process_file__is_dir(self):
temp_dir = self.filesystem.join(self._temp_dir, 'test_dir')
self.filesystem.maybe_make_directory(temp_dir)
self._file_reader.process_file(temp_dir)
# Because the log message below contains exception text, it is
# possible that the text varies across platforms. For this reason,
# we check only the portion of the log message that we control,
# namely the text at the beginning.
log_messages = self.logMessages()
# We remove the message we are looking at to prevent the tearDown()
# from raising an exception when it asserts that no log messages
# remain.
message = log_messages.pop()
self.assertTrue(
message.startswith(
"WARNING: Could not read file. Skipping: '%s'\n " % temp_dir))
self._assert_file_reader([], 1)
def test_process_file__should_not_process(self):
file_path = self._create_file('should_not_process.txt', 'contents')
self._file_reader.process_file(file_path)
self._assert_file_reader([], 1)
def test_process_file__multiple_lines(self):
file_path = self._create_file('foo.txt', 'line one\r\nline two\n')
self._file_reader.process_file(file_path)
processed = [(['line one\r', 'line two', ''], file_path, None)]
self._assert_file_reader(processed, 1)
def test_process_file__file_stdin(self):
file_path = self._create_file('-', 'file contents')
self._file_reader.process_file(file_path=file_path, test_kwarg='foo')
processed = [(['file contents'], file_path, 'foo')]
self._assert_file_reader(processed, 1)
def test_process_file__with_kwarg(self):
file_path = self._create_file('foo.txt', 'file contents')
self._file_reader.process_file(file_path=file_path, test_kwarg='foo')
processed = [(['file contents'], file_path, 'foo')]
self._assert_file_reader(processed, 1)
def test_process_paths(self):
# We test a list of paths that contains both a file and a directory.
dir = self.filesystem.join(self._temp_dir, 'foo_dir')
self.filesystem.maybe_make_directory(dir)
file_path1 = self._create_file('file1.txt', 'foo')
rel_path = self.filesystem.join('foo_dir', 'file2.txt')
file_path2 = self._create_file(rel_path, 'bar')
self._file_reader.process_paths([dir, file_path1])
processed = [(['bar'], file_path2, None), (['foo'], file_path1, None)]
self._assert_file_reader(processed, 2)
def test_count_delete_only_file(self):
self._file_reader.count_delete_only_file()
delete_only_file_count = self._file_reader.delete_only_file_count
self.assertEqual(delete_only_file_count, 1)
class RealFileSystemTest(unittest.TestCase, GenericFileSystemTests):
def setUp(self):
self.fs = FileSystem()
self.setup_generic_test_dir()
self._this_dir = os.path.dirname(os.path.abspath(__file__))
self._missing_file = os.path.join(self._this_dir, 'missing_file.py')
self._this_file = os.path.join(self._this_dir,
'filesystem_unittest.py')
def tearDown(self):
self.teardown_generic_test_dir()
self.fs = None
def test_chdir(self):
fs = FileSystem()
cwd = fs.getcwd()
newdir = '/'
if sys.platform == 'win32':
newdir = 'c:\\'
fs.chdir(newdir)
self.assertEqual(fs.getcwd(), newdir)
fs.chdir(cwd)
def test_chdir__notexists(self):
fs = FileSystem()
newdir = '/dirdoesnotexist'
if sys.platform == 'win32':
newdir = 'c:\\dirdoesnotexist'
with self.assertRaises(OSError):
fs.chdir(newdir)
def test_exists__true(self):
fs = FileSystem()
self.assertTrue(fs.exists(self._this_file))
def test_exists__false(self):
fs = FileSystem()
self.assertFalse(fs.exists(self._missing_file))
def test_getcwd(self):
fs = FileSystem()
self.assertTrue(fs.exists(fs.getcwd()))
def test_isdir__true(self):
fs = FileSystem()
self.assertTrue(fs.isdir(self._this_dir))
def test_isdir__false(self):
fs = FileSystem()
self.assertFalse(fs.isdir(self._this_file))
def test_join(self):
fs = FileSystem()
self.assertEqual(fs.join('foo', 'bar'), os.path.join('foo', 'bar'))
def test_listdir(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(fs.listdir(d), [])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(fs.listdir(d), ['foo'])
os.remove(new_file)
def test_walk(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(list(fs.walk(d)), [(d, [], [])])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(list(fs.walk(d)), [(d, [], ['foo'])])
os.remove(new_file)
def test_maybe_make_directory__success(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as base_path:
sub_path = os.path.join(base_path, 'newdir')
self.assertFalse(os.path.exists(sub_path))
self.assertFalse(fs.isdir(sub_path))
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Make sure we can re-create it.
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Clean up.
os.rmdir(sub_path)
self.assertFalse(os.path.exists(base_path))
self.assertFalse(fs.isdir(base_path))
def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform == 'win32':
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
with self.assertRaises(OSError):
fs.maybe_make_directory(sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_read_and_write_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def test_read_binary_file__missing(self):
fs = FileSystem()
with self.assertRaises(IOError):
fs.read_binary_file(self._missing_file)
def test_read_text_file__missing(self):
fs = FileSystem()
with self.assertRaises(IOError):
fs.read_text_file(self._missing_file)
def test_remove_file_with_retry(self):
RealFileSystemTest._remove_failures = 2
def remove_with_exception(filename):
RealFileSystemTest._remove_failures -= 1
if RealFileSystemTest._remove_failures >= 0:
try:
raise WindowsError
except NameError:
raise FileSystem._WindowsError
fs = FileSystem()
self.assertTrue(fs.remove('filename', remove_with_exception))
self.assertEqual(-1, RealFileSystemTest._remove_failures)
def test_sep(self):
fs = FileSystem()
self.assertEqual(fs.sep, os.sep)
self.assertEqual(fs.join('foo', 'bar'), os.path.join('foo', 'bar'))
def test_long_paths(self):
# This mostly tests UNC paths on Windows for path names > 260 chars.
# Currently, only makedirs, copyfile, and various open methods are
# verified to support UNC paths.
long_path = self.fs.join(self.generic_test_dir, 'x' * 100, 'y' * 100,
'z' * 100)
self.fs.maybe_make_directory(long_path)
file1 = self.fs.join(long_path, 'foo')
file2 = self.fs.join(long_path, 'bar')
self.fs.write_text_file(file1, 'hello')
self.fs.copyfile(file1, file2)
content = self.fs.read_text_file(file2)
self.fs.remove(file2) # No exception should be raised.
self.assertEqual(content, 'hello')
# (long_path is left in the filesystem and its removal is tested during cleanup.)
# On Windows, rmtree can handle trees containing long paths as long as
# the root is not a long path.
long_path1 = self.fs.join(self.generic_test_dir, 'a' * 100,
'b' * 100 + " 'b")
long_path2 = self.fs.join(long_path1, 'c' * 100)
self.fs.maybe_make_directory(long_path2)
file1 = self.fs.join(long_path2, 'foo')
self.fs.write_text_file(file1, 'hello')
if sys.platform == 'win32':
with self.assertRaises(AssertionError):
self.fs.rmtree(long_path2, ignore_errors=False)
else:
self.fs.rmtree(long_path2, ignore_errors=False)
self.fs.rmtree(long_path1, ignore_errors=False)
