def test_open_zip_fs_regular_filename(self):
"""Test opening a zipfs with a regular filename provided."""
from satpy.readers import FSFile
from fsspec.implementations.zip import ZipFileSystem
zip_fs = ZipFileSystem(self.zip_name)
file = FSFile(_posixify_path(self.local_filename2), zip_fs)
_assert_is_open_file_and_close(file.open())
def test_open_zip_fs_openfile(self):
"""Test opening a zipfs openfile."""
import fsspec
from satpy.readers import FSFile
open_file = fsspec.open("zip:/" + _posixify_path(self.local_filename2) + "::file://" + self.zip_name)
file = FSFile(open_file)
_assert_is_open_file_and_close(file.open())
def test_message_to_jobs_fsspec(self):
"""Test transforming a message containing filesystem specification."""
with mock.patch.dict(
'sys.modules', {
'fsspec': mock.MagicMock(),
'fsspec.spec': mock.MagicMock(),
'satpy': mock.MagicMock(),
'satpy.readers': mock.MagicMock(),
'satpy.resample': mock.MagicMock(),
'satpy.writers': mock.MagicMock(),
'satpy.dataset': mock.MagicMock(),
'satpy.version': mock.MagicMock()
}):
from fsspec.spec import AbstractFileSystem as abs_fs
from satpy.readers import FSFile as fsfile
from trollflow2.launcher import message_to_jobs
import json
filename = "/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.SEN3/Oa01_reflectance.nc" # noqa
fs = {
"cls":
"fsspec.implementations.zip.ZipFileSystem",
"protocol":
"abstract",
"args": [
"sentinel-s3-ol2wfr-zips/2020/12/10/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.zip"
], # noqa
"target_protocol":
"s3",
"target_options": {
"anon": False,
"client_kwargs": {
"endpoint_url": "https://my.dismi.se"
}
}
}
msg_data = {
"dataset": [{
"filesystem": fs,
"uid":
"zip:///S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.SEN3/Oa01_reflectance.nc::s3:///sentinel-s3-ol2wfr-zips/2020/12/10/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.zip", # noqa
"uri": "zip://" + filename
}]
}
msg = mock.MagicMock()
msg.data = msg_data
prodlist = yaml.load(yaml_test_minimal, Loader=UnsafeLoader)
jobs = message_to_jobs(msg, prodlist)
filesystemfile = jobs[999]['input_filenames'][0]
assert filesystemfile == fsfile.return_value
fsfile.assert_called_once_with(filename,
abs_fs.from_json.return_value)
abs_fs.from_json.assert_called_once_with(json.dumps(fs))
def test_sorting_fsfiles(self):
"""Test sorting FSFiles."""
from fsspec.implementations.zip import ZipFileSystem
from satpy.readers import FSFile
zip_fs = ZipFileSystem(self.zip_name)
file1 = FSFile(self.local_filename2, zip_fs)
file2 = FSFile(self.local_filename)
extra_file = os.path.normpath('/somedir/bla')
sorted_filenames = [os.fspath(file) for file in sorted([file1, file2, extra_file])]
expected_filenames = sorted([extra_file, os.fspath(file1), os.fspath(file2)])
assert sorted_filenames == expected_filenames
def _fsfiles_for_s3(input_filenames):
"""Convert S3 URLs to something Satpy can understand and use.
Examples:
Example S3 URLs (no caching):
.. code-block:: bash
polar2grid.sh ... -f s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186*
Example S3 URLs using fsspec caching:
.. code-block:: bash
polar2grid.sh ... -f simplecache::s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186*
"""
import fsspec
from satpy.readers import FSFile
kwargs = {"anon": True}
if "simplecache::" in input_filenames[0]:
kwargs = {"s3": kwargs}
for open_file in fsspec.open_files(input_filenames, **kwargs):
yield FSFile(open_file)
def test_prepare_args(sag, sge, tmp_path):
"""Test preparing arguments for getting ABI and GLM data."""
from sattools.scutil import prepare_abi_glm_ms_args
from fsspec.implementations.local import LocalFileSystem
from typhon.files.handlers.common import FileInfo
from satpy.readers import FSFile
sge.return_value = [
FileInfo(path=str(tmp_path / f"glm{i:d}"),
times=[
datetime.datetime(1900, 1, 1, 0, i),
datetime.datetime(1900, 1, 1, 0, i + 1)
],
attr={}) for i in range(5)
]
sag.return_value = [
FSFile(tmp_path / f"abi{i:d}", LocalFileSystem()) for i in range(5)
]
(gfsfs, afsfs) = prepare_abi_glm_ms_args(datetime.datetime(1900, 1, 1, 0),
datetime.datetime(1900, 1, 1, 6),
chans={8, 10},
sector="F")
assert sag.call_args[1]["sector"] == "F"
assert sge.call_args[1]["sector"] == "F"
with pytest.raises(ValueError):
prepare_abi_glm_ms_args(datetime.datetime(1900, 1, 1, 0),
datetime.datetime(1900, 1, 1, 6),
chans={8, 10},
sector="M1")
def test_fsfile_with_pathlike(self):
"""Test FSFile with path-like object."""
from pathlib import Path
from satpy.readers import FSFile
f = FSFile(Path(self.local_filename))
assert str(f) == os.fspath(f) == self.local_filename
def _filenames_to_fsfile(filenames, storage_options):
import fsspec
from satpy.readers import FSFile
if filenames:
fsspec_files = fsspec.open_files(filenames, **storage_options)
return [FSFile(f) for f in fsspec_files]
return []
def test_equality(self):
"""Test that FSFile compares equal when it should."""
from satpy.readers import FSFile
from fsspec.implementations.zip import ZipFileSystem
zip_fs = ZipFileSystem(self.zip_name)
assert FSFile(self.local_filename) == FSFile(self.local_filename)
assert (FSFile(self.local_filename,
zip_fs) == FSFile(self.local_filename, zip_fs))
assert (FSFile(self.local_filename, zip_fs) != FSFile(
self.local_filename))
assert FSFile(self.local_filename) != FSFile(self.local_filename2)
def test_generic_open_FSFile_MemoryFileSystem(self):
"""Test the generic_open method with FSFile in MemoryFileSystem."""
mem_fs = MemoryFileSystem()
mem_file = MemoryFile(fs=mem_fs,
path="{}test.DAT".format(mem_fs.root_marker),
data=b"TEST")
mem_file.commit()
fsf = FSFile(mem_file)
with hf.generic_open(fsf) as file_object:
data = file_object.read()
assert data == b'TEST'
def test_fileinfo2fspath():
"""Test converting fileinfo to fspath object."""
from typhon.files.handlers.common import FileInfo
from satpy.readers import FSFile
from sattools.tputil import fileinfo2fspath
from fsspec.implementations.local import LocalFileSystem
lfs = LocalFileSystem()
fi = FileInfo("/tmp/tofu", fs=lfs)
fsf = FSFile("/tmp/tofu", fs=lfs)
assert fileinfo2fspath(fi) == fsf
assert isinstance(fsf._file, str)
def _create_fs_file_instances(filenames, msg):
"""Create FSFile instances when filesystem is provided."""
filesystems = list(gen_dict_extract(msg.data, 'filesystem'))
if filesystems:
from satpy.readers import FSFile
from fsspec.spec import AbstractFileSystem
import json
filenames = [
FSFile(filename,
AbstractFileSystem.from_json(json.dumps(filesystem)))
for filename, filesystem in zip(filenames, filesystems)
]
return filenames
def test_read_band_FSFile(self, fsfile_open):
"""Test reading a single band from a FSFile."""
nbits = self.reader.mda['number_of_bits_per_pixel']
self.reader.filename = FSFile(
self.reader.filename) # convert str to FSFile
fsfile_open.return_value = BytesIO(
np.random.randint(0,
256,
size=int((464 * 3712 * nbits) / 8) +
self.reader.mda['total_header_length'],
dtype=np.uint8).tobytes())
res = self.reader.read_band('VIS006', None)
self.assertEqual(res.compute().shape, (464, 3712))
def test_convert_remote_files_to_fsspec_fsfile():
"""Test convertion of remote files to fsspec objects.
Case where the some of the files are already FSFile objects.
"""
from satpy.readers import FSFile
from satpy.utils import convert_remote_files_to_fsspec
filenames = [
"/tmp/file1.nc", "s3://data-bucket/file2.nc",
FSFile("ssh:///tmp/file3.nc")
]
res = convert_remote_files_to_fsspec(filenames)
assert sum([isinstance(f, FSFile) for f in res]) == 2
def test_hash(self):
"""Test that FSFile hashing behaves sanely."""
from fsspec.implementations.cached import CachingFileSystem
from fsspec.implementations.local import LocalFileSystem
from fsspec.implementations.zip import ZipFileSystem
from satpy.readers import FSFile
lfs = LocalFileSystem()
zfs = ZipFileSystem(self.zip_name)
cfs = CachingFileSystem(fs=lfs)
# make sure each name/fs-combi has its own hash
assert len({hash(FSFile(fn, fs))
for fn in {self.local_filename, self.local_filename2}
for fs in [None, lfs, zfs, cfs]}) == 2*4
def test_open_regular_file(self):
"""Test opening a regular file."""
from satpy.readers import FSFile
_assert_is_open_file_and_close(FSFile(self.local_filename).open())
def test_fsfile_with_fs_open_file_abides_pathlike(self):
"""Test that FSFile abides PathLike for fsspec OpenFile instances."""
from satpy.readers import FSFile
assert os.fspath(FSFile(self.local_file)).endswith(self.random_string)
def test_fsfile_with_pathlike(self):
from satpy.readers import FSFile
from pathlib import Path
f = FSFile(Path(self.local_filename))
assert str(f) == os.fspath(f) == self.local_filename
def test_fsfile_with_regular_filename_and_fs_spec_abides_pathlike(self):
"""Test that FSFile abides PathLike for filename+fs instances."""
from satpy.readers import FSFile
assert os.fspath(FSFile(self.random_string,
fs=None)) == self.random_string
def test_fsfile_with_regular_filename_abides_pathlike(self):
"""Test that FSFile abides PathLike for regular filenames."""
from satpy.readers import FSFile
assert os.fspath(FSFile(self.random_string)) == self.random_string
def test_regular_filename_is_returned_with_str(self):
"""Test that str give the filename."""
from satpy.readers import FSFile
assert str(FSFile(self.random_string)) == self.random_string
def test_open_local_fs_file(self):
"""Test opening a localfs file."""
from satpy.readers import FSFile
_assert_is_open_file_and_close(FSFile(self.local_file).open())
def test_get_multiscenes(sag, sge, fake_multiscene4, tmp_path):
"""Test getting a multiscene with ABI and GLM."""
from sattools.scutil import get_abi_glm_multiscenes
from sattools.abi import split_meso
from fsspec.implementations.local import LocalFileSystem
from typhon.files.handlers.common import FileInfo
from satpy.readers import FSFile
sge.return_value = [
FileInfo(path=str(tmp_path / f"glm{i:d}"),
times=[
datetime.datetime(1900, 1, 1, 0, i),
datetime.datetime(1900, 1, 1, 0, i + 1)
],
attr={}) for i in range(5)
]
sag.return_value = [
FSFile(tmp_path / f"abi{i:d}", LocalFileSystem()) for i in range(5)
]
others = list(split_meso(fake_multiscene4))
class FakeMultiScene(satpy.MultiScene):
@classmethod
def from_files(cls,
files_to_sort,
reader=None,
ensure_all_readers=False,
scene_kwargs=None,
**kwargs):
if reader == ["abi_l1b"]:
return fake_multiscene4
else:
return others[0]
with unittest.mock.patch("satpy.MultiScene", new=FakeMultiScene):
mss = list(
get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0),
datetime.datetime(1900, 1, 1, 1, 0),
chans=[8, 10],
sector="M1"))
assert "C08" in mss[0].first_scene
assert "C10" in mss[0].first_scene
assert "flash_extent_density" in mss[0].first_scene
# should be requesting GLM for the first six minutes, sector M1,
# lat/lon centred at 0
sge.assert_any_call(datetime.datetime(1900, 1, 1, 0, 0),
datetime.datetime(1900, 1, 1, 0, 6),
sector="M1",
lat=0.0,
lon=0.0)
with pytest.raises(NotImplementedError):
mss = list(
get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0),
datetime.datetime(1900, 1, 1, 1, 0),
chans=[8, 10],
sector="F"))
with pytest.raises(ValueError):
mss = list(
get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0),
datetime.datetime(1900, 1, 1, 1, 0),
chans=[8, 10],
sector="full"))
assert list(
get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0),
datetime.datetime(1900, 1, 1, 1, 0),
chans=[8, 10],
sector="M1",
limit=0)) == []
def test_repr_includes_filename(self):
"""Test that repr includes the filename."""
from satpy.readers import FSFile
assert self.random_string in repr(FSFile(self.local_file))