def setup_dir(azure):
d = str(working_dir() / 'foo')
azure.mkdir(d)
try:
yield d
finally:
azure.rm(d, recursive=True)
def setup_file(azure):
tmp = str(working_dir() / 'foo' / 'bar')
with azure.open(tmp, 'wb') as f:
f.write('123456'.encode())
try:
yield tmp
finally:
azure.rm(tmp)
def test_ls_batched(azure):
test_dir = working_dir() / 'abc'
azure.mkdir(test_dir)
with azure_teardown(azure):
test_size = 10
assert azure._ls(test_dir, batch_size=10) == []
create_files(azure, number_of_files = 10, prefix='123', root_path=test_dir)
with pytest.raises(ValueError):
assert len(azure._ls(test_dir, batch_size=1)) == test_size
assert len(azure._ls(test_dir, batch_size=9)) == test_size
assert len(azure._ls(test_dir, batch_size=10)) == test_size
assert len(azure._ls(test_dir, batch_size=11)) == test_size
assert len(azure._ls(test_dir, batch_size=2)) == test_size
assert len(azure._ls(test_dir, batch_size=100)) == test_size
assert len(azure._ls(test_dir)) == test_size
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
from contextlib import contextmanager
import os
import pytest
import shutil
import tempfile
from azure.datalake.store.core import AzureDLPath
from azure.datalake.store.multithread import ADLDownloader, ADLUploader
from tests.testing import azure, azure_teardown, md5sum, my_vcr, posix, working_dir
from azure.datalake.store.transfer import ADLTransferClient
test_dir = working_dir()
@pytest.yield_fixture()
def tempdir():
tmpdir = tempfile.mkdtemp()
try:
yield tmpdir
finally:
shutil.rmtree(tmpdir, True)
def linecount(infile):
lines = 0
with open(infile) as f:
for line in f:
def setup_env(request):
home = working_dir()
fs = AzureDLFileSystem(store_name=settings.STORE_NAME, token=settings.TOKEN)
if settings.RECORD_MODE != 'none':
if not fs.exists(home):
fs.mkdir(home)
