def test_folder_contents(self): """If input is one folder name, return a list with containg files. Folder names are not contained.""" file_name = DEMO_DIR out = get_file_names(file_name) for file in out: self.assertEqual('.', Path(file).suffix[0]) file_name = DEMO_DIR.parent out = get_file_names(file_name) for file in out: self.assertNotEqual('', Path(file).suffix)
def test_folder_contents(self): """If input is one folder name, return a list with containg files. Folder names are not contained.""" file_name = os.path.join(DATA_DIR, 'demo') out = get_file_names(file_name) for file in out: self.assertEqual('.', os.path.splitext(file)[1][0]) file_name = DATA_DIR out = get_file_names(file_name) for file in out: self.assertNotEqual('', os.path.splitext(file)[1])
def test_wrong_argument(self): """If the input contains a non-existing file, an empyt directory or a pattern that is not matched, the method should raise a ValueError.""" empty_dir = DEMO_DIR.parent with self.assertRaises(ValueError) as ve: get_file_names(str(empty_dir)) self.assertIn("no files", str(ve.exception)) no_file = 'this is not a file' with self.assertRaises(ValueError) as ve: get_file_names(no_file) self.assertIn("cannot find", str(ve.exception))
def fetch_ecmwf(self, path=None, files=None): """ Fetch and read latest ECMWF TC track predictions from the FTP dissemination server into instance. Use path argument to use local files instead. Parameters: path (str, list(str)): A location in the filesystem. Either a path to a single BUFR TC track file, or a folder containing only such files, or a globbing pattern. Passed to climada.util.files_handler.get_file_names files (file-like): An explicit list of file objects, bypassing get_file_names """ if path is None and files is None: files = self.fetch_bufr_ftp() elif files is None: files = get_file_names(path) for i, file in tqdm.tqdm(enumerate(files, 1), desc='Processing', unit='files', total=len(files)): try: file.seek(0) # reset cursor if opened file instance except AttributeError: pass self.read_one_bufr_tc(file, id_no=i) try: file.close() # discard if tempfile except AttributeError: pass
def test_folder_contents(self): """If input is one folder name, return a list with containg files. Folder names are not contained.""" file_name = DEMO_DIR out = get_file_names(file_name) self.assertGreater(len(out), 0) for file in out: self.assertTrue(Path(file).is_file())
def test_globbing(self): """If input is a glob pattern, return a list of matching visible files; omit folders. """ file_name = DEMO_DIR out = get_file_names(f'{file_name}/*') tmp_files = [ str(f) for f in Path(file_name).iterdir() if f.is_file() and not f.name.startswith('.') ] self.assertListEqual(sorted(tmp_files), sorted(out))
def test_globbing(self): """ If input is a glob pattern, return a list of matching visible files; omit folders. """ file_name = os.path.join(DATA_DIR, 'demo') out = get_file_names(file_name) tmp_files = os.listdir(file_name) tmp_files = [os.path.join(file_name, f) for f in tmp_files] tmp_files = [f for f in tmp_files if not os.path.isdir(f) and not os.path.basename(os.path.normpath(f)).startswith('.')] self.assertEqual(len(tmp_files), len(out)) self.assertEqual(sorted(tmp_files), sorted(out))
def read(self, files, descriptions='', var_names=None): """ Read and check centroids. Parameters: files (str or list(str)): absolute file name(s) or folder name containing the files to read descriptions (str or list(str), optional): one description of the data or a description of each data file var_names (dict or list(dict), default): name of the variables in the file (default: check def_source_vars() function) Raises: TypeError, ValueError """ all_files = get_file_names(files) desc_list = to_list(len(all_files), descriptions, 'descriptions') var_list = to_list(len(all_files), var_names, 'var_names') self.clear() for file, desc, var in zip(all_files, desc_list, var_list): self.append(Centroids._read_one(file, desc, var))
from rasterio import Affine from climada.hazard import Hazard from climada.entity import Exposures, ImpactFuncSet, ImpactFunc from climada.entity.exposures.base import INDICATOR_CENTR, INDICATOR_IF from climada.engine import Impact from climada.util.coordinates import write_raster from climada.util.files_handler import get_file_names logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO) HAZ_TYPE = 'FL' # hazard type abbreviation (CLIMADA convention), default: 'FL' HAZARD_PTH = '/<SET PATH>' # The directory which contains hazard files HAZ_FILES = get_file_names(HAZARD_PTH) EXP_POP_PTH = '/<SET PATH>' # The directory which contains exposure files SAVE_PTH = '/<SET PATH>' # Destinated path for the output raster files RCP = 'historical' # historical simulation HYDRO_MODEL = ['H08', 'LPJmL', 'MPI-HM', 'ORCHIDEE', 'PCR-GLOBWB', 'WaterGAP2'] # 6 * hydrological model GCM_MODEL = ['gfdl-esm2m', 'hadgem2-es', 'ipsl-cm5a-lr', 'miroc5'] # 4 * climate model YEAR = np.arange(1970,2010,10) # Years for the computation DST_META = {'width': 8640, 'height': 3347, 'crs': {'init':'epsg:4326'}, \ 'transform': Affine(0.04166666666666666, 0.0, -180.0,
def test_several_file_copy(self): """If input is a list with several file names, return the same list""" file_name = [GLB_CENTROIDS_MAT, ENT_TEMPLATE_XLS] out = get_file_names(file_name) self.assertEqual(file_name, out)
def test_one_file_copy(self): """If input is one file name, return a list with this file name""" file_name = GLB_CENTROIDS_MAT out = get_file_names(file_name) self.assertEqual([file_name], out)
def read_footprints(self, path, description=None, ref_raster=None, centroids=None, files_omit='fp_era20c_1990012515_701_0.nc'): """ Clear instance and read WISC footprints into it. Read Assumes that all footprints have the same coordinates as the first file listed/first file in dir. Parameters: path (str, list(str)): A location in the filesystem. Either a path to a single netCDF WISC footprint, or a folder containing only footprints, or a globbing pattern to one or more footprints. description (str, optional): description of the events, defaults to 'WISC historical hazard set' ref_raster (str, optional): Reference netCDF file from which to construct a new barebones Centroids instance. Defaults to the first file in path. centroids (Centroids, optional): A Centroids struct, overriding ref_raster files_omit (str, list(str), optional): List of files to omit; defaults to one duplicate storm present in the WISC set as of 2018-09-10. """ self.clear() file_names = get_file_names(path) if ref_raster is not None and centroids is not None: LOGGER.warning('Overriding ref_raster with centroids') if centroids is not None: pass elif ref_raster is not None: centroids = self._centroids_from_nc(ref_raster) elif ref_raster is None: centroids = self._centroids_from_nc(file_names[0]) if isinstance(files_omit, str): files_omit = [files_omit] LOGGER.info('Commencing to iterate over netCDF files.') for file_name in file_names: if any(fo in file_name for fo in files_omit): LOGGER.info("Omitting file %s", file_name) continue new_haz = self._read_one_nc(file_name, centroids) if new_haz is not None: self.append(new_haz) self.event_id = np.arange(1, len(self.event_id) + 1) self.frequency = np.divide(np.ones_like( self.date), (last_year(self.date) - first_year(self.date))) self.tag = TagHazard(HAZ_TYPE, 'Hazard set not saved by default', description='WISC historical hazard set.') if description is not None: self.tag.description = description
def from_footprints(cls, path, description=None, ref_raster=None, centroids=None, files_omit='fp_era20c_1990012515_701_0.nc', combine_threshold=None, intensity_thres=None): """Create new StormEurope object from WISC footprints. Assumes that all footprints have the same coordinates as the first file listed/first file in dir. Parameters ---------- path : str, list(str) A location in the filesystem. Either a path to a single netCDF WISC footprint, or a folder containing only footprints, or a globbing pattern to one or more footprints. description : str, optional description of the events, defaults to 'WISC historical hazard set' ref_raster : str, optional Reference netCDF file from which to construct a new barebones Centroids instance. Defaults to the first file in path. centroids : Centroids, optional A Centroids struct, overriding ref_raster files_omit : str, list(str), optional List of files to omit; defaults to one duplicate storm present in the WISC set as of 2018-09-10. combine_threshold : int, optional threshold for combining events in number of days. if the difference of the dates (self.date) of two events is smaller or equal to this threshold, the two events are combined into one. Default is None, Advised for WISC is 2 intensity_thres : float, optional Intensity threshold for storage in m/s. Default: class attribute StormEurope.intensity_thres (same as used by WISC SSI calculations) Returns ------- haz : StormEurope StormEurope object with data from WISC footprints. """ intensity_thres = cls.intensity_thres if intensity_thres is None else intensity_thres file_names = get_file_names(path) if ref_raster is not None and centroids is not None: LOGGER.warning('Overriding ref_raster with centroids') if centroids is not None: pass elif ref_raster is not None: centroids = cls._centroids_from_nc(ref_raster) elif ref_raster is None: centroids = cls._centroids_from_nc(file_names[0]) if isinstance(files_omit, str): files_omit = [files_omit] LOGGER.info('Commencing to iterate over netCDF files.') haz = cls() for file_name in file_names: if any(fo in file_name for fo in files_omit): LOGGER.info("Omitting file %s", file_name) continue new_haz = cls._read_one_nc(file_name, centroids, intensity_thres) if new_haz is not None: haz.append(new_haz) haz.event_id = np.arange(1, len(haz.event_id) + 1) haz.frequency = np.divide( np.ones_like(haz.date), np.max([(last_year(haz.date) - first_year(haz.date)), 1])) haz.tag = TagHazard(HAZ_TYPE, 'Hazard set not saved by default', description='WISC historical hazard set.') if description is not None: haz.tag.description = description if combine_threshold is not None: LOGGER.info('Combining events with small difference in date.') difference_date = np.diff(haz.date) for event_id_i in haz.event_id[np.append( difference_date <= combine_threshold, False)]: event_ids = [event_id_i, event_id_i + 1] haz._combine_events(event_ids) return haz