def test_storage_aws_s3_failure():
with pytest.raises(ValueError):
_ = StorageResource(
urlpath='s3://ncar-cesm-lens/lnd/monthly/',
loc_type='aws-s3',
exclude_patterns=[],
file_extension='.zarr',
fs=None,
)
def test_storage_hsi():
urlpath = '/CCSM/csm/CESM-CAM5-BGC-LE/ocn/proc/tseries/daily/SST'
loc_type = 'hsi'
exclude_dirs = []
file_extension = '.nc'
SR = StorageResource(urlpath, loc_type, exclude_dirs, file_extension)
files = SR.filelist
assert isinstance(files, list)
assert len(files) != 0
def test_storage_aws_s3():
fs = s3fs.S3FileSystem(**storage_options)
SR = StorageResource(
urlpath='s3://ncar-cesm-lens/lnd/monthly/',
loc_type='aws-s3',
exclude_patterns=[],
file_extension='.zarr',
fs=fs,
)
stores = SR.filelist
assert len(stores) != 0
def test_storage_input_file():
input_file = os.path.join(here, 'input-filelist-test.txt')
urlpath = input_file
type_ = 'input-file'
exclude_dirs = ['*/avoid-this-dir/*']
file_extension = '.nc'
SR = StorageResource(urlpath, type_, exclude_dirs, file_extension)
assert isinstance(SR, StorageResource)
files = SR.filelist
assert isinstance(files, list)
assert len(files) == 4