def test_open_zip_fs_regular_filename(self): """Test opening a zipfs with a regular filename provided.""" from satpy.readers import FSFile from fsspec.implementations.zip import ZipFileSystem zip_fs = ZipFileSystem(self.zip_name) file = FSFile(_posixify_path(self.local_filename2), zip_fs) _assert_is_open_file_and_close(file.open())
def test_open_zip_fs_openfile(self): """Test opening a zipfs openfile.""" import fsspec from satpy.readers import FSFile open_file = fsspec.open("zip:/" + _posixify_path(self.local_filename2) + "::file://" + self.zip_name) file = FSFile(open_file) _assert_is_open_file_and_close(file.open())
def test_message_to_jobs_fsspec(self): """Test transforming a message containing filesystem specification.""" with mock.patch.dict( 'sys.modules', { 'fsspec': mock.MagicMock(), 'fsspec.spec': mock.MagicMock(), 'satpy': mock.MagicMock(), 'satpy.readers': mock.MagicMock(), 'satpy.resample': mock.MagicMock(), 'satpy.writers': mock.MagicMock(), 'satpy.dataset': mock.MagicMock(), 'satpy.version': mock.MagicMock() }): from fsspec.spec import AbstractFileSystem as abs_fs from satpy.readers import FSFile as fsfile from trollflow2.launcher import message_to_jobs import json filename = "/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.SEN3/Oa01_reflectance.nc" # noqa fs = { "cls": "fsspec.implementations.zip.ZipFileSystem", "protocol": "abstract", "args": [ "sentinel-s3-ol2wfr-zips/2020/12/10/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.zip" ], # noqa "target_protocol": "s3", "target_options": { "anon": False, "client_kwargs": { "endpoint_url": "https://my.dismi.se" } } } msg_data = { "dataset": [{ "filesystem": fs, "uid": "zip:///S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.SEN3/Oa01_reflectance.nc::s3:///sentinel-s3-ol2wfr-zips/2020/12/10/S3A_OL_2_WFR____20201210T080758_20201210T080936_20201210T103707_0097_066_078_1980_MAR_O_NR_002.zip", # noqa "uri": "zip://" + filename }] } msg = mock.MagicMock() msg.data = msg_data prodlist = yaml.load(yaml_test_minimal, Loader=UnsafeLoader) jobs = message_to_jobs(msg, prodlist) filesystemfile = jobs[999]['input_filenames'][0] assert filesystemfile == fsfile.return_value fsfile.assert_called_once_with(filename, abs_fs.from_json.return_value) abs_fs.from_json.assert_called_once_with(json.dumps(fs))
def test_sorting_fsfiles(self): """Test sorting FSFiles.""" from fsspec.implementations.zip import ZipFileSystem from satpy.readers import FSFile zip_fs = ZipFileSystem(self.zip_name) file1 = FSFile(self.local_filename2, zip_fs) file2 = FSFile(self.local_filename) extra_file = os.path.normpath('/somedir/bla') sorted_filenames = [os.fspath(file) for file in sorted([file1, file2, extra_file])] expected_filenames = sorted([extra_file, os.fspath(file1), os.fspath(file2)]) assert sorted_filenames == expected_filenames
def _fsfiles_for_s3(input_filenames): """Convert S3 URLs to something Satpy can understand and use. Examples: Example S3 URLs (no caching): .. code-block:: bash polar2grid.sh ... -f s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186* Example S3 URLs using fsspec caching: .. code-block:: bash polar2grid.sh ... -f simplecache::s3://noaa-goes16/ABI-L1b-RadC/2019/001/17/*_G16_s20190011702186* """ import fsspec from satpy.readers import FSFile kwargs = {"anon": True} if "simplecache::" in input_filenames[0]: kwargs = {"s3": kwargs} for open_file in fsspec.open_files(input_filenames, **kwargs): yield FSFile(open_file)
def test_prepare_args(sag, sge, tmp_path): """Test preparing arguments for getting ABI and GLM data.""" from sattools.scutil import prepare_abi_glm_ms_args from fsspec.implementations.local import LocalFileSystem from typhon.files.handlers.common import FileInfo from satpy.readers import FSFile sge.return_value = [ FileInfo(path=str(tmp_path / f"glm{i:d}"), times=[ datetime.datetime(1900, 1, 1, 0, i), datetime.datetime(1900, 1, 1, 0, i + 1) ], attr={}) for i in range(5) ] sag.return_value = [ FSFile(tmp_path / f"abi{i:d}", LocalFileSystem()) for i in range(5) ] (gfsfs, afsfs) = prepare_abi_glm_ms_args(datetime.datetime(1900, 1, 1, 0), datetime.datetime(1900, 1, 1, 6), chans={8, 10}, sector="F") assert sag.call_args[1]["sector"] == "F" assert sge.call_args[1]["sector"] == "F" with pytest.raises(ValueError): prepare_abi_glm_ms_args(datetime.datetime(1900, 1, 1, 0), datetime.datetime(1900, 1, 1, 6), chans={8, 10}, sector="M1")
def test_fsfile_with_pathlike(self): """Test FSFile with path-like object.""" from pathlib import Path from satpy.readers import FSFile f = FSFile(Path(self.local_filename)) assert str(f) == os.fspath(f) == self.local_filename
def _filenames_to_fsfile(filenames, storage_options): import fsspec from satpy.readers import FSFile if filenames: fsspec_files = fsspec.open_files(filenames, **storage_options) return [FSFile(f) for f in fsspec_files] return []
def test_equality(self): """Test that FSFile compares equal when it should.""" from satpy.readers import FSFile from fsspec.implementations.zip import ZipFileSystem zip_fs = ZipFileSystem(self.zip_name) assert FSFile(self.local_filename) == FSFile(self.local_filename) assert (FSFile(self.local_filename, zip_fs) == FSFile(self.local_filename, zip_fs)) assert (FSFile(self.local_filename, zip_fs) != FSFile( self.local_filename)) assert FSFile(self.local_filename) != FSFile(self.local_filename2)
def test_generic_open_FSFile_MemoryFileSystem(self): """Test the generic_open method with FSFile in MemoryFileSystem.""" mem_fs = MemoryFileSystem() mem_file = MemoryFile(fs=mem_fs, path="{}test.DAT".format(mem_fs.root_marker), data=b"TEST") mem_file.commit() fsf = FSFile(mem_file) with hf.generic_open(fsf) as file_object: data = file_object.read() assert data == b'TEST'
def test_fileinfo2fspath(): """Test converting fileinfo to fspath object.""" from typhon.files.handlers.common import FileInfo from satpy.readers import FSFile from sattools.tputil import fileinfo2fspath from fsspec.implementations.local import LocalFileSystem lfs = LocalFileSystem() fi = FileInfo("/tmp/tofu", fs=lfs) fsf = FSFile("/tmp/tofu", fs=lfs) assert fileinfo2fspath(fi) == fsf assert isinstance(fsf._file, str)
def _create_fs_file_instances(filenames, msg): """Create FSFile instances when filesystem is provided.""" filesystems = list(gen_dict_extract(msg.data, 'filesystem')) if filesystems: from satpy.readers import FSFile from fsspec.spec import AbstractFileSystem import json filenames = [ FSFile(filename, AbstractFileSystem.from_json(json.dumps(filesystem))) for filename, filesystem in zip(filenames, filesystems) ] return filenames
def test_read_band_FSFile(self, fsfile_open): """Test reading a single band from a FSFile.""" nbits = self.reader.mda['number_of_bits_per_pixel'] self.reader.filename = FSFile( self.reader.filename) # convert str to FSFile fsfile_open.return_value = BytesIO( np.random.randint(0, 256, size=int((464 * 3712 * nbits) / 8) + self.reader.mda['total_header_length'], dtype=np.uint8).tobytes()) res = self.reader.read_band('VIS006', None) self.assertEqual(res.compute().shape, (464, 3712))
def test_convert_remote_files_to_fsspec_fsfile(): """Test convertion of remote files to fsspec objects. Case where the some of the files are already FSFile objects. """ from satpy.readers import FSFile from satpy.utils import convert_remote_files_to_fsspec filenames = [ "/tmp/file1.nc", "s3://data-bucket/file2.nc", FSFile("ssh:///tmp/file3.nc") ] res = convert_remote_files_to_fsspec(filenames) assert sum([isinstance(f, FSFile) for f in res]) == 2
def test_hash(self): """Test that FSFile hashing behaves sanely.""" from fsspec.implementations.cached import CachingFileSystem from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.zip import ZipFileSystem from satpy.readers import FSFile lfs = LocalFileSystem() zfs = ZipFileSystem(self.zip_name) cfs = CachingFileSystem(fs=lfs) # make sure each name/fs-combi has its own hash assert len({hash(FSFile(fn, fs)) for fn in {self.local_filename, self.local_filename2} for fs in [None, lfs, zfs, cfs]}) == 2*4
def test_open_regular_file(self): """Test opening a regular file.""" from satpy.readers import FSFile _assert_is_open_file_and_close(FSFile(self.local_filename).open())
def test_fsfile_with_fs_open_file_abides_pathlike(self): """Test that FSFile abides PathLike for fsspec OpenFile instances.""" from satpy.readers import FSFile assert os.fspath(FSFile(self.local_file)).endswith(self.random_string)
def test_fsfile_with_pathlike(self): from satpy.readers import FSFile from pathlib import Path f = FSFile(Path(self.local_filename)) assert str(f) == os.fspath(f) == self.local_filename
def test_fsfile_with_regular_filename_and_fs_spec_abides_pathlike(self): """Test that FSFile abides PathLike for filename+fs instances.""" from satpy.readers import FSFile assert os.fspath(FSFile(self.random_string, fs=None)) == self.random_string
def test_fsfile_with_regular_filename_abides_pathlike(self): """Test that FSFile abides PathLike for regular filenames.""" from satpy.readers import FSFile assert os.fspath(FSFile(self.random_string)) == self.random_string
def test_regular_filename_is_returned_with_str(self): """Test that str give the filename.""" from satpy.readers import FSFile assert str(FSFile(self.random_string)) == self.random_string
def test_open_local_fs_file(self): """Test opening a localfs file.""" from satpy.readers import FSFile _assert_is_open_file_and_close(FSFile(self.local_file).open())
def test_get_multiscenes(sag, sge, fake_multiscene4, tmp_path): """Test getting a multiscene with ABI and GLM.""" from sattools.scutil import get_abi_glm_multiscenes from sattools.abi import split_meso from fsspec.implementations.local import LocalFileSystem from typhon.files.handlers.common import FileInfo from satpy.readers import FSFile sge.return_value = [ FileInfo(path=str(tmp_path / f"glm{i:d}"), times=[ datetime.datetime(1900, 1, 1, 0, i), datetime.datetime(1900, 1, 1, 0, i + 1) ], attr={}) for i in range(5) ] sag.return_value = [ FSFile(tmp_path / f"abi{i:d}", LocalFileSystem()) for i in range(5) ] others = list(split_meso(fake_multiscene4)) class FakeMultiScene(satpy.MultiScene): @classmethod def from_files(cls, files_to_sort, reader=None, ensure_all_readers=False, scene_kwargs=None, **kwargs): if reader == ["abi_l1b"]: return fake_multiscene4 else: return others[0] with unittest.mock.patch("satpy.MultiScene", new=FakeMultiScene): mss = list( get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0), datetime.datetime(1900, 1, 1, 1, 0), chans=[8, 10], sector="M1")) assert "C08" in mss[0].first_scene assert "C10" in mss[0].first_scene assert "flash_extent_density" in mss[0].first_scene # should be requesting GLM for the first six minutes, sector M1, # lat/lon centred at 0 sge.assert_any_call(datetime.datetime(1900, 1, 1, 0, 0), datetime.datetime(1900, 1, 1, 0, 6), sector="M1", lat=0.0, lon=0.0) with pytest.raises(NotImplementedError): mss = list( get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0), datetime.datetime(1900, 1, 1, 1, 0), chans=[8, 10], sector="F")) with pytest.raises(ValueError): mss = list( get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0), datetime.datetime(1900, 1, 1, 1, 0), chans=[8, 10], sector="full")) assert list( get_abi_glm_multiscenes(datetime.datetime(1900, 1, 1, 0, 0), datetime.datetime(1900, 1, 1, 1, 0), chans=[8, 10], sector="M1", limit=0)) == []
def test_repr_includes_filename(self): """Test that repr includes the filename.""" from satpy.readers import FSFile assert self.random_string in repr(FSFile(self.local_file))