def test_init_should_set_tmp_dir_path_to_temporary_directory_file_path(mocked_boto3, mocked_tempfile, mocked_urlparse): mocked_tempfile('/tmp') mocked_urlparse((None, 'bucket', '/')) result = S3Walker(fs_url) assert result.tmp_dir_path == '/tmp'
def test_init_should_request_client_from_boto3(mocked_boto3_client, mocked_urlparse): mock_client = mocked_boto3_client() mocked_urlparse((None, 'bucket', 'path/')) S3Walker(fs_url) mock_client.assert_called_with('s3')
def test_init_should_set_client_to_boto3_client(mocked_boto3_client, mocked_urlparse): mocked_boto3_client({}) mocked_urlparse((None, 'bucket', 'path/')) result = S3Walker(fs_url) assert result.client == {}
def test_listdir_should_yield_directories_if_they_exist_without_path(mocked_boto3_pagination, mocked_urlparse): mocked_boto3_pagination([{'CommonPrefixes': [ {'Prefix': 'dir1/'}, {'Prefix': 'dir2/'} ]}]) mocked_urlparse((None, 'bucket', '/')) walker = S3Walker(fs_url) directories = [] for directory in walker._listdir('/'): directories.append(directory) assert len(directories) == 2 assert directories[0].created is None assert directories[0].is_dir is True assert directories[0].is_link is False assert directories[0].modified is None assert directories[0].name == 'dir1' assert directories[0].size is None assert directories[1].created is None assert directories[1].is_dir is True assert directories[1].is_link is False assert directories[1].modified is None assert directories[1].name == 'dir2' assert directories[1].size is None
def test_listdir_should_yield_files_if_they_exist_without_path(mocked_boto3_pagination, mocked_urlparse): mocked_boto3_pagination([{'Contents': [ {'LastModified': datetime.date(1970, 1, 1), 'Key': 'file1.txt', 'Size': 1000}, {'LastModified': datetime.date(1970, 1, 2), 'Key': 'file2.txt', 'Size': 2000} ]}]) mocked_urlparse((None, 'bucket', '/')) walker = S3Walker(fs_url) files = [] for file in walker._listdir('/'): files.append(file) assert len(files) == 2 assert files[0].created is None assert files[0].is_dir is False assert files[0].is_link is False assert files[0].modified == datetime.date(1970, 1, 1) assert files[0].name == 'file1.txt' assert files[0].size == 1000 assert files[1].created is None assert files[1].is_dir is False assert files[1].is_link is False assert files[1].modified == datetime.date(1970, 1, 2) assert files[1].name == 'file2.txt' assert files[1].size == 2000
def test_init_should_request_mkdtemp_from_tempfile(mocked_boto3, mocked_tempfile, mocked_urlparse): mock_tempfile = mocked_tempfile() mocked_urlparse((None, 'bucket', 'path/')) S3Walker(fs_url) mock_tempfile.mkdtemp.assert_called()
def test_listdir_should_request_get_paginator_from_client_with_path_without_ending_slash(mocked_boto3, mocked_urlparse): mocked_urlparse((None, 'bucket', 'path/')) walker = S3Walker(fs_url) next(walker._listdir('/path'), None) mocked_boto3.client.return_value.get_paginator.assert_called_with('list_objects')
def test_listdir_should_request_paginate_from_paginator_with_root_path(mocked_boto3_pagination, mocked_urlparse): mock_paginator = mocked_boto3_pagination() mocked_urlparse((None, 'bucket', '/')) walker = S3Walker(fs_url) next(walker._listdir('/'), None) mock_paginator.paginate.assert_called_with(Bucket='bucket', Prefix='', Delimiter='/')
def test_open_should_throw_if_file_if_resource_not_found_exception_is_thrown(mocked_boto3, mocked_open_return, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_open_return(True, fs.errors.ResourceNotFound('not found')) mocked_urlparse((None, 'bucket', '/path/')) walker = S3Walker(fs_url) with pytest.raises(FileNotFoundError, match=r'File /dir1/file.txt not found'): walker.open('/dir1/file.txt')
def test_open_should_return_result_from_open(mocked_boto3, mocked_open_return, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_open_return(False, {}) mocked_urlparse((None, 'bucket', 'path/')) walker = S3Walker(fs_url) result = walker.open('/') assert result == {}
def test_close_should_set_tmp_dir_path_to_none_if_tmp_dir_path_exists(mocked_boto3, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_tempfile('/tmp') mocked_urlparse((None, 'bucket', 'path/')) s3_walker = S3Walker(fs_url) s3_walker.close() assert s3_walker.tmp_dir_path is None
def test_close_should_not_request_rmtree_from_shutil_if_tmp_dir_path_does_not_exist(mocked_boto3, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_tempfile() mocked_urlparse((None, 'bucket', 'path/')) s3_walker = S3Walker(fs_url) s3_walker.close() mocked_shutil.rmtree.assert_not_called()
def test_open_should_request_open(mocked_boto3, mocked_open, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_urlparse((None, 'bucket', '/path/')) mocked_os.path.join = mock.MagicMock(return_value='/tmp/path/dir1/file.txt') walker = S3Walker(fs_url) walker.tmp_dir_path = '/tmp' walker.open('/dir1/file.txt') mocked_open.assert_called_with('/tmp/path/dir1/file.txt', 'rb')
def test_listdir_should_not_yield_files_if_they_do_not_exist_without_path(mocked_boto3_pagination, mocked_urlparse): mocked_boto3_pagination([{'CommonPrefixes': []}]) mocked_urlparse((None, 'bucket', '/')) walker = S3Walker(fs_url) files = [] for file in walker._listdir('/'): files.append(file) assert len(files) == 0
def test_listdir_should_not_yield_directories_if_they_do_not_exist_with_path(mocked_boto3_pagination, mocked_urlparse): mocked_boto3_pagination([{'CommonPrefixes': []}]) mocked_urlparse((None, 'bucket', 'path/')) walker = S3Walker(fs_url) directories = [] for directory in walker._listdir('/path'): directories.append(directory) assert len(directories) == 0
def test_open_should_not_request_download_file_from_boto3_client_if_file_exists(mocked_boto3, mocked_open, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_urlparse((None, 'bucket', '/path/')) mocked_os.path.isfile = mock.MagicMock(return_value=True) walker = S3Walker(fs_url) walker.tmp_dir_path = '/tmp' walker.open('/dir1/file.txt') walker.client.download_file.assert_not_called()
def test_open_should_request_makedirs_from_os_if_file_does_not_exist(mocked_boto3, mocked_open, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_urlparse((None, 'bucket', '/')) mocked_os.path.isfile = mock.MagicMock(return_value=False) mocked_os.path.join = mock.MagicMock(return_value='/tmp/dir1/dir2') walker = S3Walker(fs_url) walker.tmp_dir_path = '/tmp' walker.open('/dir1/dir2/file.txt') mocked_os.makedirs.assert_called_with('/tmp/dir1/dir2', exist_ok=True)
def test_open_should_request_download_file_from_boto3_client_if_file_does_not_exist(mocked_boto3, mocked_open, mocked_os, mocked_shutil, mocked_tempfile, mocked_urlparse): mocked_urlparse((None, 'bucket', '/path/')) mocked_os.path.isfile = mock.MagicMock(return_value=False) mocked_os.path.join = mock.MagicMock(side_effect=['/tmp/path/dir1/file.txt', 'path/dir1/file.txt']) walker = S3Walker(fs_url) walker.tmp_dir_path = '/tmp' walker.open('/dir1/file.txt') walker.client.download_file.assert_called_with('bucket', 'path/dir1/file.txt', '/tmp/path/dir1/file.txt')
def test_get_fs_url_should_return_fs_url(): walker = S3Walker(fs_url) result = walker.get_fs_url() assert result == fs_url
def test_init_should_request_urlparse(mocked_boto3, mocked_urlparse): mock_urlparse = mocked_urlparse((None, 'bucket', 'path/')) S3Walker(fs_url) mock_urlparse.assert_called_with('s3://bucket/path/')
def test_init_should_set_bucket_to_urlparse_bucket(mocked_boto3, mocked_urlparse): mocked_urlparse((None, 'bucket', 'path/')) result = S3Walker(fs_url) assert result.bucket == 'bucket'
def test_init_should_set_fs_url(mocked_boto3, mocked_urlparse): mocked_urlparse((None, 'bucket', 'path/')) result = S3Walker(fs_url) assert result.fs_url == fs_url
def test_init_should_set_root_to_empty_string_if_url_path_is_empty(mocked_boto3, mocked_urlparse): mocked_urlparse((None, 'bucket', '/')) result = S3Walker(fs_url) assert result.root == ''
def test_init_should_set_root_to_sanitized_path_value_if_url_path_is_not_empty(mocked_boto3, mocked_urlparse): mocked_urlparse((None, 'bucket', 'path/')) result = S3Walker(fs_url) assert result.root == 'path'