Example #1
0
def test_init_should_set_tmp_dir_path_to_temporary_directory_file_path(mocked_boto3, mocked_tempfile, mocked_urlparse):
    mocked_tempfile('/tmp')
    mocked_urlparse((None, 'bucket', '/'))

    result = S3Walker(fs_url)

    assert result.tmp_dir_path == '/tmp'
Example #2
0
def test_init_should_request_client_from_boto3(mocked_boto3_client, mocked_urlparse):
    mock_client = mocked_boto3_client()
    mocked_urlparse((None, 'bucket', 'path/'))

    S3Walker(fs_url)

    mock_client.assert_called_with('s3')
Example #3
0
def test_init_should_set_client_to_boto3_client(mocked_boto3_client, mocked_urlparse):
    mocked_boto3_client({})
    mocked_urlparse((None, 'bucket', 'path/'))

    result = S3Walker(fs_url)

    assert result.client == {}
Example #4
0
def test_listdir_should_yield_directories_if_they_exist_without_path(mocked_boto3_pagination, mocked_urlparse):
    mocked_boto3_pagination([{'CommonPrefixes': [
        {'Prefix': 'dir1/'},
        {'Prefix': 'dir2/'}
    ]}])
    mocked_urlparse((None, 'bucket', '/'))
    walker = S3Walker(fs_url)

    directories = []
    for directory in walker._listdir('/'):
        directories.append(directory)

    assert len(directories) == 2
    assert directories[0].created is None
    assert directories[0].is_dir is True
    assert directories[0].is_link is False
    assert directories[0].modified is None
    assert directories[0].name == 'dir1'
    assert directories[0].size is None
    assert directories[1].created is None
    assert directories[1].is_dir is True
    assert directories[1].is_link is False
    assert directories[1].modified is None
    assert directories[1].name == 'dir2'
    assert directories[1].size is None
Example #5
0
def test_listdir_should_yield_files_if_they_exist_without_path(mocked_boto3_pagination, mocked_urlparse):
    mocked_boto3_pagination([{'Contents': [
        {'LastModified': datetime.date(1970, 1, 1), 'Key': 'file1.txt', 'Size': 1000},
        {'LastModified': datetime.date(1970, 1, 2), 'Key': 'file2.txt', 'Size': 2000}
    ]}])
    mocked_urlparse((None, 'bucket', '/'))
    walker = S3Walker(fs_url)

    files = []
    for file in walker._listdir('/'):
        files.append(file)

    assert len(files) == 2
    assert files[0].created is None
    assert files[0].is_dir is False
    assert files[0].is_link is False
    assert files[0].modified == datetime.date(1970, 1, 1)
    assert files[0].name == 'file1.txt'
    assert files[0].size == 1000
    assert files[1].created is None
    assert files[1].is_dir is False
    assert files[1].is_link is False
    assert files[1].modified == datetime.date(1970, 1, 2)
    assert files[1].name == 'file2.txt'
    assert files[1].size == 2000
Example #6
0
def test_init_should_request_mkdtemp_from_tempfile(mocked_boto3, mocked_tempfile, mocked_urlparse):
    mock_tempfile = mocked_tempfile()
    mocked_urlparse((None, 'bucket', 'path/'))

    S3Walker(fs_url)

    mock_tempfile.mkdtemp.assert_called()
Example #7
0
def test_listdir_should_request_get_paginator_from_client_with_path_without_ending_slash(mocked_boto3, mocked_urlparse):
    mocked_urlparse((None, 'bucket', 'path/'))
    walker = S3Walker(fs_url)

    next(walker._listdir('/path'), None)

    mocked_boto3.client.return_value.get_paginator.assert_called_with('list_objects')
Example #8
0
def test_listdir_should_request_paginate_from_paginator_with_root_path(mocked_boto3_pagination, mocked_urlparse):
    mock_paginator = mocked_boto3_pagination()
    mocked_urlparse((None, 'bucket', '/'))
    walker = S3Walker(fs_url)

    next(walker._listdir('/'), None)

    mock_paginator.paginate.assert_called_with(Bucket='bucket', Prefix='', Delimiter='/')
Example #9
0
def test_open_should_throw_if_file_if_resource_not_found_exception_is_thrown(mocked_boto3, mocked_open_return,
                                                                             mocked_os, mocked_shutil, mocked_tempfile,
                                                                             mocked_urlparse):
    mocked_open_return(True, fs.errors.ResourceNotFound('not found'))
    mocked_urlparse((None, 'bucket', '/path/'))
    walker = S3Walker(fs_url)

    with pytest.raises(FileNotFoundError, match=r'File /dir1/file.txt not found'):
        walker.open('/dir1/file.txt')
Example #10
0
def test_open_should_return_result_from_open(mocked_boto3, mocked_open_return, mocked_os, mocked_shutil,
                                             mocked_tempfile, mocked_urlparse):
    mocked_open_return(False, {})
    mocked_urlparse((None, 'bucket', 'path/'))
    walker = S3Walker(fs_url)

    result = walker.open('/')

    assert result == {}
Example #11
0
def test_close_should_set_tmp_dir_path_to_none_if_tmp_dir_path_exists(mocked_boto3, mocked_shutil, mocked_tempfile,
                                                                      mocked_urlparse):
    mocked_tempfile('/tmp')
    mocked_urlparse((None, 'bucket', 'path/'))
    s3_walker = S3Walker(fs_url)

    s3_walker.close()

    assert s3_walker.tmp_dir_path is None
Example #12
0
def test_close_should_not_request_rmtree_from_shutil_if_tmp_dir_path_does_not_exist(mocked_boto3, mocked_shutil,
                                                                                    mocked_tempfile, mocked_urlparse):
    mocked_tempfile()
    mocked_urlparse((None, 'bucket', 'path/'))
    s3_walker = S3Walker(fs_url)

    s3_walker.close()

    mocked_shutil.rmtree.assert_not_called()
Example #13
0
def test_open_should_request_open(mocked_boto3, mocked_open, mocked_os, mocked_shutil, mocked_tempfile,
                                  mocked_urlparse):
    mocked_urlparse((None, 'bucket', '/path/'))
    mocked_os.path.join = mock.MagicMock(return_value='/tmp/path/dir1/file.txt')
    walker = S3Walker(fs_url)
    walker.tmp_dir_path = '/tmp'

    walker.open('/dir1/file.txt')

    mocked_open.assert_called_with('/tmp/path/dir1/file.txt', 'rb')
Example #14
0
def test_listdir_should_not_yield_files_if_they_do_not_exist_without_path(mocked_boto3_pagination, mocked_urlparse):
    mocked_boto3_pagination([{'CommonPrefixes': []}])
    mocked_urlparse((None, 'bucket', '/'))
    walker = S3Walker(fs_url)

    files = []
    for file in walker._listdir('/'):
        files.append(file)

    assert len(files) == 0
Example #15
0
def test_listdir_should_not_yield_directories_if_they_do_not_exist_with_path(mocked_boto3_pagination, mocked_urlparse):
    mocked_boto3_pagination([{'CommonPrefixes': []}])
    mocked_urlparse((None, 'bucket', 'path/'))
    walker = S3Walker(fs_url)

    directories = []
    for directory in walker._listdir('/path'):
        directories.append(directory)

    assert len(directories) == 0
Example #16
0
def test_open_should_not_request_download_file_from_boto3_client_if_file_exists(mocked_boto3, mocked_open, mocked_os,
                                                                                mocked_shutil, mocked_tempfile,
                                                                                mocked_urlparse):
    mocked_urlparse((None, 'bucket', '/path/'))
    mocked_os.path.isfile = mock.MagicMock(return_value=True)
    walker = S3Walker(fs_url)
    walker.tmp_dir_path = '/tmp'

    walker.open('/dir1/file.txt')

    walker.client.download_file.assert_not_called()
Example #17
0
def test_open_should_request_makedirs_from_os_if_file_does_not_exist(mocked_boto3, mocked_open, mocked_os,
                                                                     mocked_shutil, mocked_tempfile, mocked_urlparse):
    mocked_urlparse((None, 'bucket', '/'))
    mocked_os.path.isfile = mock.MagicMock(return_value=False)
    mocked_os.path.join = mock.MagicMock(return_value='/tmp/dir1/dir2')
    walker = S3Walker(fs_url)
    walker.tmp_dir_path = '/tmp'

    walker.open('/dir1/dir2/file.txt')

    mocked_os.makedirs.assert_called_with('/tmp/dir1/dir2', exist_ok=True)
Example #18
0
def test_open_should_request_download_file_from_boto3_client_if_file_does_not_exist(mocked_boto3, mocked_open,
                                                                                    mocked_os, mocked_shutil,
                                                                                    mocked_tempfile, mocked_urlparse):
    mocked_urlparse((None, 'bucket', '/path/'))
    mocked_os.path.isfile = mock.MagicMock(return_value=False)
    mocked_os.path.join = mock.MagicMock(side_effect=['/tmp/path/dir1/file.txt', 'path/dir1/file.txt'])
    walker = S3Walker(fs_url)
    walker.tmp_dir_path = '/tmp'

    walker.open('/dir1/file.txt')

    walker.client.download_file.assert_called_with('bucket', 'path/dir1/file.txt', '/tmp/path/dir1/file.txt')
Example #19
0
def test_get_fs_url_should_return_fs_url():
    walker = S3Walker(fs_url)

    result = walker.get_fs_url()

    assert result == fs_url
Example #20
0
def test_init_should_request_urlparse(mocked_boto3, mocked_urlparse):
    mock_urlparse = mocked_urlparse((None, 'bucket', 'path/'))

    S3Walker(fs_url)

    mock_urlparse.assert_called_with('s3://bucket/path/')
Example #21
0
def test_init_should_set_bucket_to_urlparse_bucket(mocked_boto3, mocked_urlparse):
    mocked_urlparse((None, 'bucket', 'path/'))

    result = S3Walker(fs_url)

    assert result.bucket == 'bucket'
Example #22
0
def test_init_should_set_fs_url(mocked_boto3, mocked_urlparse):
    mocked_urlparse((None, 'bucket', 'path/'))

    result = S3Walker(fs_url)

    assert result.fs_url == fs_url
Example #23
0
def test_init_should_set_root_to_empty_string_if_url_path_is_empty(mocked_boto3, mocked_urlparse):
    mocked_urlparse((None, 'bucket', '/'))

    result = S3Walker(fs_url)

    assert result.root == ''
Example #24
0
def test_init_should_set_root_to_sanitized_path_value_if_url_path_is_not_empty(mocked_boto3, mocked_urlparse):
    mocked_urlparse((None, 'bucket', 'path/'))

    result = S3Walker(fs_url)

    assert result.root == 'path'