def test_convert(self): expected = ['aa', 'b*', 'cc'] actual = VarNamesLike.convert('aa,b*,cc') self.assertEqual(actual, expected) with self.assertRaises(ValidationError) as err: VarNamesLike.convert(['aa', 1, 'bb']) self.assertEqual(str(err.exception), 'List of variables names expected.') self.assertEqual(None, VarNamesLike.convert(None))
def test_convert(self): expected = ['aa', 'b*', 'cc'] actual = VarNamesLike.convert('aa,b*,cc') self.assertEqual(actual, expected) with self.assertRaises(ValueError) as err: VarNamesLike.convert(['aa', 1, 'bb']) self.assertTrue('string or a list' in str(err.exception)) self.assertEqual(None, VarNamesLike.convert(None))
def test_convert(self): expected = ['aa', 'b*', 'cc'] actual = VarNamesLike.convert('aa,b*,cc') self.assertEqual(actual, expected) with self.assertRaises(ValidationError) as err: VarNamesLike.convert(['aa', 1, 'bb']) self.assertEqual(str(err.exception), 'List of variables names expected.') self.assertEqual(None, VarNamesLike.convert(None))
def test_make_local(self): data_source = self._local_data_store.query('local_w_temporal')[0] with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'from_local_to_local' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds = data_source.make_local(new_ds_title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), TimeRangeLike.convert( (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)))) new_ds_2_title = 'from_local_to_local_var' new_ds_2_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds_2_vars = VarNamesLike.convert(['sm']) new_ds_w_one_variable = data_source.make_local(new_ds_2_title, time_range=new_ds_2_time_range, var_names=new_ds_2_vars) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_2_title) data_set = new_ds_w_one_variable.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'}) new_ds_3_title = 'from_local_to_local_range' new_ds_3_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds_3_vars = VarNamesLike.convert(['sm']) new_ds_3_region = PolygonLike.convert("10,10,20,20") new_ds_w_region = data_source.make_local(new_ds_3_title, time_range=new_ds_3_time_range, var_names=new_ds_3_vars, region=new_ds_3_region) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_3_title) self.assertEqual(new_ds_w_region.spatial_coverage(), PolygonLike.convert("10,10,20,20")) data_set = new_ds_w_region.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'}) no_data = data_source.make_local('no_data', time_range=(datetime.datetime(2020, 11, 14, 0, 0), datetime.datetime(2020, 11, 15, 23, 59))) self.assertIsNone(no_data)
def data_frame_subset(gdf: gpd.GeoDataFrame, region_op: bool = 'intersects', region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None) -> gpd.GeoDataFrame: """ Create a GeoDataFrame subset from given variables (data frame columns) and/or region. :param gdf: A GeoDataFrame. :param region_op: The geometric operation to be performed if *region* is given. :param region: A region polygon used to filter rows. :param var_names: The variables (columns) to select. :return: A GeoDataFrame subset. """ region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) if not var_names and not region: return gdf if var_names: if 'geometry' not in var_names: var_names = ['geometry'] + var_names gdf = gdf[var_names] if region and region_op: geom_str = PolygonLike.format(region) gdf = data_frame_query(gdf, f'@{region_op}("{geom_str}")') return gdf
def select_var(ds: xr.Dataset, var: VarNamesLike.TYPE = None) -> xr.Dataset: """ Filter the dataset, by leaving only the desired variables in it. The original dataset information, including original coordinates, is preserved. :param ds: The dataset from which to perform selection. :param var: One or more variable names to select and preserve in the dataset. \ All of these are valid 'var_name' 'var_name1,var_name2,var_name3' ['var_name1', 'var_name2']. \ One can also use wildcards when doing the selection. E.g., choosing 'var_name*' for selection \ will select all variables that start with 'var_name'. This can be used to select variables \ along with their auxiliary variables, to select all uncertainty variables, and so on. :return: A filtered dataset """ if not var: return ds var_names = VarNamesLike.convert(var) dropped_var_names = list(ds.data_vars.keys()) for pattern in var_names: keep = fnmatch.filter(dropped_var_names, pattern) for name in keep: dropped_var_names.remove(name) return ds.drop(dropped_var_names)
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None var_names = VarNamesLike.convert(var_names) if var_names else None selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'CCI Open Data Portal data source "{}"\ndoes not seem to have any datasets'.format(self.id) if time_range is not None: msg += ' in given time range {}'.format(TimeRangeLike.format(time_range)) raise DataAccessError(msg) files = self._get_urls_list(selected_file_list, _ODP_PROTOCOL_OPENDAP) try: ds = open_xarray_dataset(files) if region: ds = normalize_impl(ds) ds = subset_spatial_impl(ds, region) if var_names: ds = ds.drop([var_name for var_name in ds.data_vars.keys() if var_name not in var_names]) return ds except OSError as e: if time_range: raise DataAccessError("Cannot open remote dataset for time range {}:\n" "{}" .format(TimeRangeLike.format(time_range), e), source=self) from e else: raise DataAccessError("Cannot open remote dataset:\n" "{}" .format(TimeRangeLike.format(time_range), e), source=self) from e
def data_frame_subset(gdf: gpd.GeoDataFrame, region_op: bool = 'intersects', region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None) -> gpd.GeoDataFrame: """ Create a GeoDataFrame subset from given variables (data frame columns) and/or region. :param gdf: A GeoDataFrame. :param region_op: The geometric operation to be performed if *region* is given. :param region: A region polygon used to filter rows. :param var_names: The variables (columns) to select. :return: A GeoDataFrame subset. """ region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) if not var_names and not region: return gdf if var_names: if 'geometry' not in var_names: var_names = ['geometry'] + var_names gdf = gdf[var_names] if region and region_op: geom_str = PolygonLike.format(region) gdf = data_frame_query(gdf, f'@{region_op}("{geom_str}")') return gdf
def read_netcdf(file: str, drop_variables: VarNamesLike.TYPE = None, decode_cf: bool = True, normalize: bool = True, decode_times: bool = True, engine: str = None) -> xr.Dataset: """ Read a dataset from a netCDF 3/4 or HDF file. :param file: The netCDF file path. :param drop_variables: List of variables to be dropped. :param decode_cf: Whether to decode CF attributes and coordinate variables. :param normalize: Whether to normalize the dataset's geo- and time-coding upon opening. See operation ``normalize``. :param decode_times: Whether to decode time information (convert time coordinates to ``datetime`` objects). :param engine: Optional netCDF engine name. """ drop_variables = VarNamesLike.convert(drop_variables) ds = xr.open_dataset(file, drop_variables=drop_variables, decode_cf=decode_cf, decode_times=decode_times, engine=engine) chunks = get_spatial_ext_chunk_sizes(ds) if chunks and 'time' in ds.dims: chunks['time'] = 1 if chunks: ds = ds.chunk(chunks) if normalize: return adjust_temporal_attrs(normalize_op(ds)) return ds
def select_var(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None) -> xr.Dataset: """ Filter the dataset, by leaving only the desired variables in it. The original dataset information, including original coordinates, is preserved. :param ds: The dataset or dataframe from which to perform selection. :param var: One or more variable names to select and preserve in the dataset. \ All of these are valid 'var_name' 'var_name1,var_name2,var_name3' ['var_name1', 'var_name2']. \ One can also use wildcards when doing the selection. E.g., choosing 'var_name*' for selection \ will select all variables that start with 'var_name'. This can be used to select variables \ along with their auxiliary variables, to select all uncertainty variables, and so on. :return: A filtered dataset """ if not var: return ds ds = DatasetLike.convert(ds) var_names = VarNamesLike.convert(var) dropped_var_names = list(ds.data_vars.keys()) for pattern in var_names: keep = fnmatch.filter(dropped_var_names, pattern) for name in keep: dropped_var_names.remove(name) return ds.drop(dropped_var_names)
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None if region: region = PolygonLike.convert(region) if var_names: var_names = VarNamesLike.convert(var_names) paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance( time_series[i], datetime ) and time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if paths: paths = sorted(set(paths)) try: ds = open_xarray_dataset(paths) if region: ds = normalize_impl(ds) ds = subset_spatial_impl(ds, region) if var_names: ds = ds.drop([ var_name for var_name in ds.data_vars.keys() if var_name not in var_names ]) return ds except OSError as e: if time_range: raise DataAccessError( "Cannot open local dataset for time range {}:\n" "{}".format(TimeRangeLike.format(time_range), e), source=self) from e else: raise DataAccessError("Cannot open local dataset:\n" "{}".format(e), source=self) from e else: if time_range: raise DataAccessError( "No local datasets available for\nspecified time range {}". format(TimeRangeLike.format(time_range)), source=self) else: raise DataAccessError("No local datasets available", source=self)
def detect_outliers(ds: xr.Dataset, var: VarNamesLike.TYPE, threshold_low: float = 0.05, threshold_high: float = 0.95, quantiles: bool = True, mask: bool = False) -> xr.Dataset: """ Detect outliers in the given Dataset. When mask=True the input dataset should not contain nan values, otherwise all existing nan values will be marked as 'outliers' in the mask data array added to the output dataset. :param ds: The dataset for which to do outlier detection :param var: Variable or variables in the dataset to which to do outlier detection. Note that when multiple variables are selected, absolute threshold values might not make much sense. Wild cards can be used to select multiple variables matching a pattern. :param threshold_low: Values less or equal to this will be removed/masked :param threshold_high: Values greater or equal to this will be removed/masked :bool quantiles: If True, threshold values are treated as quantiles, otherwise as absolute values. :bool mask: If True, an ancillary variable containing flag values for outliers will be added to the dataset. Otherwise, outliers will be replaced with nan directly in the data variables. :return: The dataset with outliers masked or replaced with nan """ # Create a list of variable names on which to perform outlier detection # based on the input comma separated list that can contain wildcards var_patterns = VarNamesLike.convert(var) all_vars = list(ds.data_vars.keys()) variables = list() for pattern in var_patterns: leave = fnmatch.filter(all_vars, pattern) variables = variables + leave # For each array in the dataset for which we should detect outliers, detect # outliers ret_ds = ds.copy() for var_name in variables: if quantiles: # Get threshold values threshold_low = ret_ds[var_name].quantile(threshold_low) threshold_high = ret_ds[var_name].quantile(threshold_high) # If not mask, put nans in the data arrays for min/max outliers if not mask: arr = ret_ds[var_name] attrs = arr.attrs ret_ds[var_name] = arr.where((arr > threshold_low) & (arr < threshold_high)) ret_ds[var_name].attrs = attrs else: # Create and add a data variable containing the mask for this data # variable _mask_outliers(ret_ds, var_name, threshold_low, threshold_high) return ret_ds
def make_local(self, local_name: str, local_id: str = None, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> Optional[DataSource]: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: add_to_data_store_registry() local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: raise ValueError('Cannot initialize `local` DataStore') _uuid = LocalDataStore.generate_uuid(ref_id=self.id, time_range=time_range, region=region, var_names=var_names) if not local_name or len(local_name) == 0: local_name = "local.{}.{}".format(self.id, _uuid) existing_ds_list = local_store.query(ds_id=local_name) if len(existing_ds_list) == 1: return existing_ds_list[0] else: existing_ds_list = local_store.query(ds_id='local.%s' % local_name) if len(existing_ds_list) == 1: if existing_ds_list[0].meta_info.get('uuid', None) == _uuid: return existing_ds_list[0] else: raise ValueError('Datastore {} already contains dataset {}'.format(local_store.id, local_name)) local_meta_info = self.meta_info.copy() local_meta_info['ref_uuid'] = local_meta_info.get('uuid', None) local_meta_info['uuid'] = _uuid local_ds = local_store.create_data_source(local_name, region, local_name, time_range=time_range, var_names=var_names, meta_info=self.meta_info.copy()) if local_ds: if not local_ds.is_complete: self._make_local(local_ds, time_range, region, var_names, monitor=monitor) if local_ds.is_empty: local_store.remove_data_source(local_ds) return None local_store.register_ds(local_ds) return local_ds return None
def long_term_average(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Create a 'mean over years' dataset by averaging the values of the given input dataset over all years. The output is a climatological dataset with the same resolution as the input dataset. E.g. a daily input dataset will create a daily climatology consisting of 365 days, a monthly input dataset will create a monthly climatology, etc. Seasonal input datasets must have matching seasons over all years denoted by the same date each year. E.g., first date of each quarter. The output dataset will then be a seasonal climatology where each season is denoted with the same date as in the input dataset. For further information on climatological datasets, see http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#climatological-statistics :param ds: A dataset to average :param var: If given, only these variables will be preserved in the resulting dataset :param monitor: A progress monitor :return: A climatological long term average dataset """ ds = DatasetLike.convert(ds) # Check if time dtype is what we want if 'datetime64[ns]' != ds.time.dtype: raise ValidationError( 'Long term average operation expects a dataset with the' ' time coordinate of type datetime64[ns], but received' ' {}. Running the normalize operation on this' ' dataset may help'.format(ds.time.dtype)) try: t_resolution = ds.attrs['time_coverage_resolution'] except KeyError: raise ValidationError( 'Could not determine temporal resolution. Running' ' the adjust_temporal_attrs operation beforehand may' ' help.') var = VarNamesLike.convert(var) # Shallow if var: ds = select_var(ds, var) if t_resolution == 'P1D': return _lta_daily(ds) elif t_resolution == 'P1M': return _lta_monthly(ds, monitor) else: return _lta_general(ds, monitor)
def reduce(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None, dim: DimNamesLike.TYPE = None, method: str = 'mean', monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Reduce the given variables of the given dataset along the given dimensions. If no variables are given, all variables of the dataset will be reduced. If no dimensions are given, all dimensions will be reduced. If no variables have been given explicitly, it can be set that only variables featuring numeric values should be reduced. :param ds: Dataset to reduce :param var: Variables in the dataset to reduce :param dim: Dataset dimensions along which to reduce :param method: reduction method :param monitor: A progress monitor """ ufuncs = { 'min': np.nanmin, 'max': np.nanmax, 'mean': np.nanmean, 'median': np.nanmedian, 'sum': np.nansum } ds = DatasetLike.convert(ds) if not var: var = list(ds.data_vars.keys()) var_names = VarNamesLike.convert(var) if not dim: dim = list(ds.coords.keys()) else: dim = DimNamesLike.convert(dim) retset = ds.copy() for var_name in var_names: intersection = [ value for value in dim if value in retset[var_name].dims ] with monitor.starting("Reduce dataset", total_work=100): monitor.progress(5) with monitor.child(95).observing("Reduce"): retset[var_name] = retset[var_name].reduce(ufuncs[method], dim=intersection, keep_attrs=True) return retset
def long_term_average(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Create a 'mean over years' dataset by averaging the values of the given input dataset over all years. The output is a climatological dataset with the same resolution as the input dataset. E.g. a daily input dataset will create a daily climatology consisting of 365 days, a monthly input dataset will create a monthly climatology, etc. Seasonal input datasets must have matching seasons over all years denoted by the same date each year. E.g., first date of each quarter. The output dataset will then be a seasonal climatology where each season is denoted with the same date as in the input dataset. For further information on climatological datasets, see http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#climatological-statistics :param ds: A dataset to average :param var: If given, only these variables will be preserved in the resulting dataset :param monitor: A progress monitor :return: A climatological long term average dataset """ ds = DatasetLike.convert(ds) # Check if time dtype is what we want if 'datetime64[ns]' != ds.time.dtype: raise ValidationError('Long term average operation expects a dataset with the' ' time coordinate of type datetime64[ns], but received' ' {}. Running the normalize operation on this' ' dataset may help'.format(ds.time.dtype)) try: t_resolution = ds.attrs['time_coverage_resolution'] except KeyError: raise ValidationError('Could not determine temporal resolution. Running' ' the adjust_temporal_attrs operation beforehand may' ' help.') var = VarNamesLike.convert(var) # Shallow retset = ds.copy() if var: retset = select_var(retset, var) if t_resolution == 'P1D': return _lta_daily(retset, monitor) elif t_resolution == 'P1M': return _lta_monthly(retset, monitor) else: return _lta_general(retset, monitor)
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None, monitor: Monitor = Monitor.NONE) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None var_names = VarNamesLike.convert(var_names) if var_names else None paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance(time_series[i], datetime) and time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if not paths: raise self._empty_error(time_range) paths = sorted(set(paths)) try: excluded_variables = self._meta_info.get('exclude_variables') if excluded_variables: drop_variables = [variable.get('name') for variable in excluded_variables] else: drop_variables = None # TODO: combine var_names and drop_variables return open_xarray_dataset(paths, region=region, var_names=var_names, drop_variables=drop_variables, monitor=monitor) except HTTPError as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e) from e except (URLError, socket.timeout) as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e, error_cls=NetworkError) from e except OSError as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e) from e
def __init__(self, ds_id: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, meta_info: dict = None, status: DataSourceStatus = None): self._id = ds_id if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert( temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert( tuple([ files_range[0][0], files_range[files_number - 1][1] ])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert( (files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert( spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert(variables) if variables else [] self._meta_info = meta_info if meta_info else OrderedDict() if self._variables and not self._meta_info.get('variables', None): self._meta_info['variables'] = [{ 'name': var_name, 'units': '', 'long_name': '', 'standard_name': '' } for var_name in self._variables] self._status = status if status else DataSourceStatus.READY
def read_zarr(path: str, key: str = None, secret: str = None, token: str = None, drop_variables: VarNamesLike.TYPE = None, decode_cf: bool = True, decode_times: bool = True, normalize: bool = True) -> xr.Dataset: """ Read a dataset from a Zarr directory, Zarr ZIP archive, or remote Zarr object storage. For the Zarr format, refer to http://zarr.readthedocs.io/en/stable/. :param path: Zarr directory path, Zarr ZIP archive path, or (S3) object storage URL. :param key: Optional (AWS) access key identifier. Valid only if *path* is a URL. :param secret: Optional (AWS) secret access key. Valid only if *path* is a URL. :param token: Optional (AWS) access token. Valid only if *path* is a URL. :param drop_variables: List of variables to be dropped. :param decode_cf: Whether to decode CF attributes and coordinate variables. :param decode_times: Whether to decode time information (convert time coordinates to ``datetime`` objects). :param normalize: Whether to normalize the dataset's geo- and time-coding upon opening. See operation ``normalize``. """ drop_variables = VarNamesLike.convert(drop_variables) is_s3_url = path.startswith('s3://') is_http_url = path.startswith('http://') or path.startswith('https://') if is_s3_url or is_http_url: root = path client_kwargs = None if is_http_url: url = urllib.parse.urlparse(path) root = url.path[1:] if url.path.startswith('/') else url.path client_kwargs = dict(endpoint_url=f'{url.scheme}://{url.netloc}') store = s3fs.S3Map(root, s3=s3fs.S3FileSystem(anon=not (key or secret or token), key=key, secret=secret, token=token, client_kwargs=client_kwargs)) else: store = path ds = xr.open_zarr(store, drop_variables=drop_variables, decode_cf=decode_cf, decode_times=decode_times) if normalize: return adjust_temporal_attrs(normalize_op(ds)) return ds
def read_netcdf(file: str, drop_variables: VarNamesLike.TYPE = None, decode_cf: bool = True, decode_times: bool = True, engine: str = None) -> xr.Dataset: """ Read a dataset from a netCDF 3/4 or HDF file. :param file: The netCDF file path. :param drop_variables: List of variables to be dropped. :param decode_cf: Whether to decode CF attributes and coordinate variables. :param decode_times: Whether to decode time information (convert time coordinates to ``datetime`` objects). :param engine: Optional netCDF engine name. """ drop_variables = VarNamesLike.convert(drop_variables) return xr.open_dataset(file, drop_variables=drop_variables, decode_cf=decode_cf, decode_times=decode_times, engine=engine)
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None if region: region = PolygonLike.convert(region) if var_names: var_names = VarNamesLike.convert(var_names) paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance(time_series[i], datetime) and \ time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if paths: paths = sorted(set(paths)) try: ds = open_xarray_dataset(paths) if region: [lat_min, lon_min, lat_max, lon_max] = region.bounds ds = ds.sel(drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) if var_names: ds = ds.drop([ var_name for var_name in ds.variables.keys() if var_name not in var_names ]) return ds except OSError as e: raise IOError("Files: {} caused:\nOSError({}): {}".format( paths, e.errno, e.strerror)) else: return None
def __init__(self, ds_id: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, meta_info: dict = None, status: DataSourceStatus = None): self._id = ds_id if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert(temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert(tuple([files_range[0][0], files_range[files_number - 1][1]])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert((files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert(spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert(variables) if variables else [] self._meta_info = meta_info if meta_info else OrderedDict() if self._variables and not self._meta_info.get('variables', None): self._meta_info['variables'] = [ {'name': var_name, 'units': '', 'long_name': '', 'standard_name': '' } for var_name in self._variables] self._status = status if status else DataSourceStatus.READY
def reduce(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None, dim: DimNamesLike.TYPE = None, method: str = 'mean', monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Reduce the given variables of the given dataset along the given dimensions. If no variables are given, all variables of the dataset will be reduced. If no dimensions are given, all dimensions will be reduced. If no variables have been given explicitly, it can be set that only variables featuring numeric values should be reduced. :param ds: Dataset to reduce :param var: Variables in the dataset to reduce :param dim: Dataset dimensions along which to reduce :param method: reduction method :param monitor: A progress monitor """ ufuncs = {'min': np.nanmin, 'max': np.nanmax, 'mean': np.nanmean, 'median': np.nanmedian, 'sum': np.nansum} ds = DatasetLike.convert(ds) if not var: var = list(ds.data_vars.keys()) var_names = VarNamesLike.convert(var) if not dim: dim = list(ds.coords.keys()) else: dim = DimNamesLike.convert(dim) retset = ds.copy() for var_name in var_names: intersection = [value for value in dim if value in retset[var_name].dims] with monitor.starting("Reduce dataset", total_work=100): monitor.progress(5) with monitor.child(95).observing("Reduce"): retset[var_name] = retset[var_name].reduce(ufuncs[method], dim=intersection, keep_attrs=True) return retset
def __init__(self, name: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, reference_type: str = None, reference_name: str = None): self._name = name if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert( temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert( tuple([ files_range[0][0], files_range[files_number - 1][1] ])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert( (files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert( spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert( variables) if variables else None self._reference_type = reference_type if reference_type else None self._reference_name = reference_name
def read_zarr(path: str, file_system: str = 'Local', drop_variables: VarNamesLike.TYPE = None, decode_cf: bool = True, decode_times: bool = True, normalize: bool = True) -> xr.Dataset: """ Read a dataset from a Zarr directory, Zarr ZIP archive, or remote Zarr object storage. For the Zarr format, refer to http://zarr.readthedocs.io/en/stable/. :param path: Zarr directory path, Zarr ZIP archive path, or object storage path or bucket name. :param file_system: File system identifier, "Local" is your locally mounted file system, for Amazon S3 use "S3", for general Object Storage use "OBS". :param drop_variables: List of variables to be dropped. :param decode_cf: Whether to decode CF attributes and coordinate variables. :param decode_times: Whether to decode time information (convert time coordinates to ``datetime`` objects). :param normalize: Whether to normalize the dataset's geo- and time-coding upon opening. See operation ``normalize``. """ drop_variables = VarNamesLike.convert(drop_variables) if file_system == 'Local': ds = xr.open_zarr(path, drop_variables=drop_variables, decode_cf=decode_cf, decode_times=decode_times) elif file_system == 'S3' or file_system == 'OBS': import s3fs store = s3fs.S3Map(path, s3=(s3fs.S3FileSystem(anon=True))) ds = xr.open_zarr(store, drop_variables=drop_variables, decode_cf=decode_cf, decode_times=decode_times) else: raise ValidationError(f'Unknown file_system {file_system!r}') if normalize: return adjust_temporal_attrs(normalize_op(ds)) return ds
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'Data source \'{}\' does not seem to have any data files'.format( self.name) if time_range is not None: msg += ' in given time range {}'.format( TimeRangeLike.format(time_range)) raise IOError(msg) files = self._get_urls_list(selected_file_list, _ODP_PROTOCOL_OPENDAP) try: ds = open_xarray_dataset(files) if region: [lat_min, lon_min, lat_max, lon_max] = region.bounds ds = ds.sel(drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) if var_names: ds = ds.drop([ var_name for var_name in ds.variables.keys() if var_name not in var_names ]) return ds except OSError as e: raise IOError("Files: {} caused:\nOSError({}): {}".format( files, e.errno, e.strerror))
def make_local(self, local_name: str, local_id: str = None, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> Optional[DataSource]: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None ds_id = local_name title = local_id local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: add_to_data_store_registry() local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: raise ValueError('Cannot initialize `local` DataStore') uuid = LocalDataStore.generate_uuid(ref_id=self.id, time_range=time_range, region=region, var_names=var_names) if not ds_id or len(ds_id) == 0: ds_id = "local.{}.{}".format(self.id, uuid) existing_ds_list = local_store.query(ds_id=ds_id) if len(existing_ds_list) == 1: return existing_ds_list[0] else: existing_ds_list = local_store.query(ds_id='local.%s' % ds_id) if len(existing_ds_list) == 1: if existing_ds_list[0].meta_info.get('uuid', None) == uuid: return existing_ds_list[0] else: raise ValueError( 'Datastore {} already contains dataset {}'.format( local_store.id, ds_id)) local_meta_info = self.meta_info.copy() local_meta_info['ref_uuid'] = local_meta_info.get('uuid', None) local_meta_info['uuid'] = uuid local_ds = local_store.create_data_source(ds_id, title=title, time_range=time_range, region=region, var_names=var_names, meta_info=local_meta_info, lock_file=True) if local_ds: if not local_ds.is_complete: try: self._make_local(local_ds, time_range, region, var_names, monitor=monitor) except Cancellation as c: local_store.remove_data_source(local_ds) raise c except Exception as e: if local_ds.is_empty: local_store.remove_data_source(local_ds) raise e if local_ds.is_empty: local_store.remove_data_source(local_ds) return None local_store.register_ds(local_ds) return local_ds else: return None
def _make_local(self, local_ds: LocalDataSource, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): local_id = local_ds.id time_range = TimeRangeLike.convert(time_range) region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) time_range, region, var_names = self._apply_make_local_fixes( time_range, region, var_names) compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False do_update_of_verified_time_coverage_start_once = True verified_time_coverage_start = None verified_time_coverage_end = None encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) if region or var_names: protocol = _ODP_PROTOCOL_OPENDAP else: protocol = _ODP_PROTOCOL_HTTP local_path = os.path.join(local_ds.data_store.data_store_path, local_id) if not os.path.exists(local_path): os.makedirs(local_path) selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'CCI Open Data Portal data source "{}"\ndoes not seem to have any datasets'.format( self.id) if time_range is not None: msg += ' in given time range {}'.format( TimeRangeLike.format(time_range)) raise DataAccessError(msg) try: if protocol == _ODP_PROTOCOL_OPENDAP: do_update_of_variables_meta_info_once = True do_update_of_region_meta_info_once = True files = self._get_urls_list(selected_file_list, protocol) monitor.start('Sync ' + self.id, total_work=len(files)) for idx, dataset_uri in enumerate(files): child_monitor = monitor.child(work=1) file_name = os.path.basename(dataset_uri) local_filepath = os.path.join(local_path, file_name) time_coverage_start = selected_file_list[idx][1] time_coverage_end = selected_file_list[idx][2] try: child_monitor.start(label=file_name, total_work=1) remote_dataset = xr.open_dataset(dataset_uri) if var_names: remote_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.data_vars.keys() if var_name not in var_names ]) if region: remote_dataset = normalize_impl(remote_dataset) remote_dataset = subset_spatial_impl( remote_dataset, region) geo_lon_min, geo_lat_min, geo_lon_max, geo_lat_max = region.bounds remote_dataset.attrs[ 'geospatial_lat_min'] = geo_lat_min remote_dataset.attrs[ 'geospatial_lat_max'] = geo_lat_max remote_dataset.attrs[ 'geospatial_lon_min'] = geo_lon_min remote_dataset.attrs[ 'geospatial_lon_max'] = geo_lon_max if do_update_of_region_meta_info_once: local_ds.meta_info['bbox_maxx'] = geo_lon_max local_ds.meta_info['bbox_minx'] = geo_lon_min local_ds.meta_info['bbox_maxy'] = geo_lat_max local_ds.meta_info['bbox_miny'] = geo_lat_min do_update_of_region_meta_info_once = False if compression_enabled: for sel_var_name in remote_dataset.variables.keys( ): remote_dataset.variables.get( sel_var_name).encoding.update( encoding_update) remote_dataset.to_netcdf(local_filepath) child_monitor.progress(work=1, msg=str(time_coverage_start)) finally: if do_update_of_variables_meta_info_once: variables_info = local_ds.meta_info.get( 'variables', []) local_ds.meta_info['variables'] = [ var_info for var_info in variables_info if var_info.get('name') in remote_dataset. variables.keys() and var_info.get( 'name') not in remote_dataset.dims.keys() ] do_update_of_variables_meta_info_once = False local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) if do_update_of_verified_time_coverage_start_once: verified_time_coverage_start = time_coverage_start do_update_of_verified_time_coverage_start_once = False verified_time_coverage_end = time_coverage_end child_monitor.done() else: outdated_file_list = [] for file_rec in selected_file_list: filename, _, _, file_size, url = file_rec dataset_file = os.path.join(local_path, filename) # todo (forman, 20160915): must perform better checks on dataset_file if it is... # ... outdated or incomplete or corrupted. # JSON also includes "checksum" and "checksum_type" fields. if not os.path.isfile(dataset_file) or ( file_size and os.path.getsize(dataset_file) != file_size): outdated_file_list.append(file_rec) if outdated_file_list: with monitor.starting('Sync ' + self.id, len(outdated_file_list)): bytes_to_download = sum( [file_rec[3] for file_rec in outdated_file_list]) dl_stat = _DownloadStatistics(bytes_to_download) file_number = 1 for filename, coverage_from, coverage_to, file_size, url in outdated_file_list: dataset_file = os.path.join(local_path, filename) sub_monitor = monitor.child(work=1.0) # noinspection PyUnusedLocal def reporthook(block_number, read_size, total_file_size): dl_stat.handle_chunk(read_size) sub_monitor.progress(work=read_size, msg=str(dl_stat)) sub_monitor_msg = "file %d of %d" % ( file_number, len(outdated_file_list)) with sub_monitor.starting(sub_monitor_msg, file_size): urllib.request.urlretrieve( url[protocol], filename=dataset_file, reporthook=reporthook) file_number += 1 local_ds.add_dataset( os.path.join(local_id, filename), (coverage_from, coverage_to)) if do_update_of_verified_time_coverage_start_once: verified_time_coverage_start = coverage_from do_update_of_verified_time_coverage_start_once = False verified_time_coverage_end = coverage_to except OSError as e: raise DataAccessError( "Copying remote data source failed: {}".format(e), source=self) from e local_ds.meta_info['temporal_coverage_start'] = TimeLike.format( verified_time_coverage_start) local_ds.meta_info['temporal_coverage_end'] = TimeLike.format( verified_time_coverage_end) local_ds.save(True)
def long_term_average(source: str, year_min: int, year_max: int, file: str, var: VarNamesLike.TYPE = None, save: bool = False, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Perform the long term monthly average of the given monthly or daily data source for the given range of years. Depending on the given year range, data size, as well as internet connection quality, this operation can potentially take a very long time to finish. Careful consideration is needed in choosing the var parameter to create meaningful outputs. This is unique for each data source. :param source: The data source from which to extract the monthly average :param year_min: The earliest year of the desired time range :param year_max: The most recent year of the desired time range :param file: filepath where to save the long term average dataset :param var: If given, only these variable names will be preserved in the output. :param save: If True, saves the data downloaded during this operation. This can potentially be a very large amount of data. :param monitor: A progress monitor to use :return: The Long Term Average dataset. """ var = VarNamesLike.convert(var) n_years = year_max - year_min + 1 res = 0 total_work = 100 # Select the appropriate data source data_store_list = DATA_STORE_REGISTRY.get_data_stores() data_sources = query_data_sources(data_store_list, name=source) if len(data_sources) == 0: raise ValueError("No data_source found for the given query\ term {}".format(source)) elif len(data_sources) > 1: raise ValueError("{} data_sources found for the given query\ term {}".format(data_sources, source)) data_source = data_sources[0] source_info = data_source.cache_info # Check if we have a monthly data source fq = data_source.meta_info['time_frequency'] if fq != 'mon': raise ValueError("Only monthly datasets are supported for time being.") with monitor.starting('LTA', total_work=total_work): # Set up the monitor monitor.progress(work=0) step = total_work * 0.9 / n_years # Process the data source year by year year = year_min while year != year_max + 1: tmin = "{}-01-01".format(year) tmax = "{}-12-31".format(year) # Determine if the data for the given year are already downloaded # If at least one file of the given time range is present, we # don't delete the data for this year, we do the syncing anyway. was_already_downloaded = False dt_range = to_datetime_range(tmin, tmax) for date in source_info: if dt_range[0] <= date <= dt_range[1]: was_already_downloaded = True # One is enough break worked = monitor._worked data_source.sync(dt_range, monitor=monitor.child(work=step * 0.9)) if worked == monitor._worked: monitor.progress(work=step * 0.9) ds = data_source.open_dataset(dt_range) # Filter the dataset ds = select_var(ds, var) try: if res == 0: res = ds / n_years else: # Xarray doesn't do automatic alignment for in place # operations, hence we have to do it manually res = res + ds.reindex_like(res) / n_years except TypeError: raise TypeError('One or more data arrays feature a dtype that\ can not be divided. Consider using the var\ parameter to filter the dataset.') ds.close() # delete data for the current year, if it should be deleted and it # was not already downloaded. if (not save) and (not was_already_downloaded): data_source.delete_local(dt_range) monitor.progress(work=step * 0.1) year = year + 1 monitor.progress(msg='Saving the LTA dataset') save_dataset(res, file) monitor.progress(total_work * 0.1) return res
def temporal_agg(source: str, start_date: str = None, end_date: str = None, var: VarNamesLike.TYPE = None, level: str = 'mon', method: str = 'mean', save_data: bool = False, monitor: Monitor = Monitor.NONE) -> (xr.Dataset, str): """ Perform temporal aggregation of the given data source to the given level using the given method for the given time range. Only full time periods of the given time range will be aggregated. Depending on the given time range, data size, as well as internet connection quality, this operation can potentially take a very long time to finish. Careful consideration is needed in choosing the var parameter to create meaningful outputs. This is unique for each data source. The aggregation result is saved into the local data store for later reuse. :param source: Data source to aggregate :param start_date: Start date of aggregation. If not given, data source start date is used instead :param end_date: End date of aggregation. If not given, data source end date is used instead :param var: If given, only these dataset variables will be preserved in the result :param level: Aggregation level :param method: Aggregation method :param save_data: Whether to save data downloaded during this operation. This can potentially be a lot of data. :param monitor: A progress monitor to use :return: The local data source identifier for the aggregated data """ # Raise not implemented, while not finished raise ValueError("Operation is not implemented.") var = VarNamesLike.convert(var) # Select the appropriate data source data_store_list = DATA_STORE_REGISTRY.get_data_stores() data_sources = query_data_sources(data_store_list, name=source) if len(data_sources) == 0: raise ValueError("No data_source found for the given query " "term {}".format(source)) elif len(data_sources) > 1: raise ValueError("{} data_sources found for the given query " "term {}".format(data_sources, source)) data_source = data_sources[0] source_info = data_source.cache_info # We have to do this to have temporal coverage info in meta_info data_source._init_file_list() # Check if the data source temporal resolution is known known_res = ('day', '8-days', 'mon', 'yr') fq = data_source.meta_info['time_frequency'] if (not fq) or (fq not in known_res): raise ValueError("The given data source features unknown time " "resolution: {}".format(fq)) # Check if the operation supports the desired aggregation step valid_steps = list() valid_steps.append(('day', 'mon')) if (fq, level) not in valid_steps: raise ValueError("Currently the operation does not support aggregation" " from {} to {}".format(fq, level)) # Determine start and end dates if not start_date: start_date = data_source.meta_info['temporal_coverage_start'] start_date = to_datetime(start_date) # If start_date is not start of the month, move it to the 1st of next # month if start_date.day != 1: try: start_date = datetime(start_date.year, start_date.month + 1, 1) except ValueError: # We have tried to set the month to 13 start_date = datetime(start_date.year + 1, 1, 1) if not end_date: end_date = data_source.meta_info['temporal_coverage_end'] end_date = to_datetime(end_date) # If end date is not end of the month, move it to the last day of the # previous month if not _is_end_of_month(end_date): try: end_date = datetime(end_date.year, end_date.month - 1, 27) except ValueError: # We have tried to set the month to 0 end_date = datetime(end_date.year - 1, 12, 31) end_date = _end_of_month(end_date.year, end_date.month) # Determine the count of processing periods n_periods = (end_date.year - start_date.year + 1) * 12\ + end_date.month - start_date.month - 11 # 2000-4-1, 2000-6-30 -> 12 + 2 -11 = 3 if n_periods < 1: raise ValueError("The given time range does not contain any full " "calendar months to do aggregation with.") # Set up the monitor total_work = 100 with monitor.starting('Aggregate', total_work=total_work): monitor.progress(work=0) step = total_work * 0.9 / n_periods # Process the data source period by period tmin = start_date while tmin < end_date: tmax = _end_of_month(tmin.year, tmin.month) # Determine if the data for the given period are already downloaded # If at least one file of the given time range is present, we # don't delete the data for this period, we do the syncing anyway was_already_downloaded = False dt_range = to_datetime_range(tmin, tmax) for date in source_info: if dt_range[0] <= date <= dt_range[1]: was_already_downloaded = True # One is enough break worked = monitor._worked data_source.sync(dt_range, monitor=monitor.child(work=step * 0.9)) if worked == monitor._worked: monitor.progress(work=step * 0.9) ds = data_source.open_dataset(dt_range) # Filter the dataset ds = select_var(ds, var) # Do the aggregation # Save the dataset for this period into local data store # Close and delete the files if needed ds.close() # delete data for the current period,if it should be deleted and it # was not already downloaded. if (not save_data) and (not was_already_downloaded): data_source.delete_local(dt_range) monitor.progress(work=step * 0.1) # tmin for next iteration try: tmin = datetime(tmin.year, tmin.month + 1, 1) except ValueError: # Couldn't add a month -> end of year tmin = datetime(tmin.year + 1, 1, 1) pass monitor.progress(work=step * 0.1) # Return the local data source id return None
def _make_local(self, local_ds: 'LocalDataSource', time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): local_id = local_ds.id time_range = TimeRangeLike.convert(time_range) if time_range else None var_names = VarNamesLike.convert(var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({'zlib': True, 'complevel': compression_level}) local_path = os.path.join(local_ds.data_store.data_store_path, local_id) data_store_path = local_ds.data_store.data_store_path if not os.path.exists(local_path): os.makedirs(local_path) monitor.start("Sync " + self.id, total_work=len(self._files.items())) for remote_relative_filepath, coverage in self._files.items(): child_monitor = monitor.child(work=1) file_name = os.path.basename(remote_relative_filepath) local_relative_filepath = os.path.join(local_id, file_name) local_absolute_filepath = os.path.join(data_store_path, local_relative_filepath) remote_absolute_filepath = os.path.join(self._data_store.data_store_path, remote_relative_filepath) if isinstance(coverage, Tuple): time_coverage_start = coverage[0] time_coverage_end = coverage[1] if not time_range or time_coverage_start >= time_range[0] and time_coverage_end <= time_range[1]: if region or var_names: do_update_of_variables_meta_info_once = True do_update_of_region_meta_info_once = True remote_dataset = None try: remote_dataset = xr.open_dataset(remote_absolute_filepath) if var_names: remote_dataset = remote_dataset.drop( [var_name for var_name in remote_dataset.data_vars.keys() if var_name not in var_names]) if region: remote_dataset = normalize_impl(remote_dataset) remote_dataset = adjust_spatial_attrs_impl(subset_spatial_impl(remote_dataset, region), allow_point=False) if do_update_of_region_meta_info_once: # subset_spatial_impl local_ds.meta_info['bbox_maxx'] = remote_dataset.attrs['geospatial_lon_max'] local_ds.meta_info['bbox_minx'] = remote_dataset.attrs['geospatial_lon_min'] local_ds.meta_info['bbox_maxy'] = remote_dataset.attrs['geospatial_lat_max'] local_ds.meta_info['bbox_miny'] = remote_dataset.attrs['geospatial_lat_min'] do_update_of_region_meta_info_once = False if compression_enabled: for sel_var_name in remote_dataset.variables.keys(): remote_dataset.variables.get(sel_var_name).encoding.update(encoding_update) remote_dataset.to_netcdf(local_absolute_filepath) child_monitor.progress(work=1, msg=str(time_coverage_start)) finally: if do_update_of_variables_meta_info_once and remote_dataset is not None: variables_info = local_ds.meta_info.get('variables', []) local_ds.meta_info['variables'] = [var_info for var_info in variables_info if var_info.get('name') in remote_dataset.variables.keys() and var_info.get('name') not in remote_dataset.dims.keys()] # noinspection PyUnusedLocal do_update_of_variables_meta_info_once = False local_ds.add_dataset(os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) child_monitor.done() else: shutil.copy(remote_absolute_filepath, local_absolute_filepath) local_ds.add_dataset(local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() monitor.done() return local_id
def _make_local(self, local_ds: LocalDataSource, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): # local_name = local_ds.name local_id = local_ds.name time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) if region or var_names: protocol = _ODP_PROTOCOL_OPENDAP else: protocol = _ODP_PROTOCOL_HTTP local_path = os.path.join(local_ds.data_store.data_store_path, local_id) if not os.path.exists(local_path): os.makedirs(local_path) selected_file_list = self._find_files(time_range) if protocol == _ODP_PROTOCOL_OPENDAP: files = self._get_urls_list(selected_file_list, protocol) monitor.start('Sync ' + self.name, total_work=len(files)) for idx, dataset_uri in enumerate(files): child_monitor = monitor.child(work=1) file_name = os.path.basename(dataset_uri) local_filepath = os.path.join(local_path, file_name) time_coverage_start = selected_file_list[idx][1] time_coverage_end = selected_file_list[idx][2] remote_netcdf = None local_netcdf = None try: remote_netcdf = NetCDF4DataStore(dataset_uri) local_netcdf = NetCDF4DataStore(local_filepath, mode='w', persist=True) local_netcdf.set_attributes(remote_netcdf.get_attrs()) remote_dataset = xr.Dataset.load_store(remote_netcdf) process_region = False if region: geo_lat_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_min') geo_lat_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_max') geo_lon_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_min') geo_lon_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_max') geo_lat_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_resolution') geo_lon_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_resolution') if not (isnan(geo_lat_min) or isnan(geo_lat_max) or isnan(geo_lon_min) or isnan(geo_lon_max) or isnan(geo_lat_res) or isnan(geo_lon_res)): process_region = True [lat_min, lon_min, lat_max, lon_max] = region.bounds lat_min = floor( (lat_min - geo_lat_min) / geo_lat_res) lat_max = ceil( (lat_max - geo_lat_min) / geo_lat_res) lon_min = floor( (lon_min - geo_lon_min) / geo_lon_res) lon_max = ceil( (lon_max - geo_lon_min) / geo_lon_res) # TODO (kbernat): check why dataset.sel fails! remote_dataset = remote_dataset.isel( drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) geo_lat_max = lat_max * geo_lat_res + geo_lat_min geo_lat_min += lat_min * geo_lat_res geo_lon_max = lon_max * geo_lon_res + geo_lon_min geo_lon_min += lon_min * geo_lon_res if not var_names: var_names = [ var_name for var_name in remote_netcdf.variables.keys() ] var_names.extend([ coord_name for coord_name in remote_dataset.coords.keys() if coord_name not in var_names ]) child_monitor.start(label=file_name, total_work=len(var_names)) for sel_var_name in var_names: var_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.variables.keys() if var_name != sel_var_name ]) if compression_enabled: var_dataset.variables.get( sel_var_name).encoding.update(encoding_update) local_netcdf.store_dataset(var_dataset) child_monitor.progress(work=1, msg=sel_var_name) if process_region: local_netcdf.set_attribute('geospatial_lat_min', geo_lat_min) local_netcdf.set_attribute('geospatial_lat_max', geo_lat_max) local_netcdf.set_attribute('geospatial_lon_min', geo_lon_min) local_netcdf.set_attribute('geospatial_lon_max', geo_lon_max) finally: if remote_netcdf: remote_netcdf.close() if local_netcdf: local_netcdf.close() local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) child_monitor.done() else: outdated_file_list = [] for file_rec in selected_file_list: filename, _, _, file_size, url = file_rec dataset_file = os.path.join(local_path, filename) # todo (forman, 20160915): must perform better checks on dataset_file if it is... # ... outdated or incomplete or corrupted. # JSON also includes "checksum" and "checksum_type" fields. if not os.path.isfile(dataset_file) or ( file_size and os.path.getsize(dataset_file) != file_size): outdated_file_list.append(file_rec) if outdated_file_list: with monitor.starting('Sync ' + self.name, len(outdated_file_list)): bytes_to_download = sum( [file_rec[3] for file_rec in outdated_file_list]) dl_stat = _DownloadStatistics(bytes_to_download) file_number = 1 for filename, coverage_from, coverage_to, file_size, url in outdated_file_list: if monitor.is_cancelled(): raise InterruptedError dataset_file = os.path.join(local_path, filename) sub_monitor = monitor.child(work=1.0) # noinspection PyUnusedLocal def reporthook(block_number, read_size, total_file_size): dl_stat.handle_chunk(read_size) if monitor.is_cancelled(): raise InterruptedError sub_monitor.progress(work=read_size, msg=str(dl_stat)) sub_monitor_msg = "file %d of %d" % ( file_number, len(outdated_file_list)) with sub_monitor.starting(sub_monitor_msg, file_size): urllib.request.urlretrieve(url[protocol], filename=dataset_file, reporthook=reporthook) file_number += 1 local_ds.add_dataset(os.path.join(local_id, filename), (coverage_from, coverage_to)) local_ds.save() monitor.done()
def long_term_average(ds: DatasetLike.TYPE, var: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Perform long term average of the given dataset by doing a mean of monthly values over the time range covered by the dataset. E.g. it averages all January values, all February values, etc, to create a dataset with twelve time slices each containing a mean of respective monthly values. For further information on climatological datasets, see http://cfconventions.org/cf-conventions/v1.6.0/cf-conventions.html#climatological-statistics :param ds: A monthly dataset to average :param var: If given, only these variables will be preserved in the resulting dataset :param monitor: A progress monitor :return: A climatological long term average dataset """ ds = DatasetLike.convert(ds) # Check if time dtype is what we want if 'datetime64[ns]' != ds.time.dtype: raise ValueError( 'Long term average operation expects a dataset with the' ' time coordinate of type datetime64[ns], but received' ' {}. Running the normalize operation on this' ' dataset may help'.format(ds.time.dtype)) # Check if we have a monthly dataset try: if ds.attrs['time_coverage_resolution'] != 'P1M': raise ValueError( 'Long term average operation expects a monthly dataset' ' running temporal aggregation on this dataset' ' beforehand may help.') except KeyError: raise ValueError('Could not determine temporal resolution. Running' ' the adjust_temporal_attrs operation beforehand may' ' help.') var = VarNamesLike.convert(var) # Shallow retset = ds.copy() if var: retset = select_var(retset, var) time_min = pd.Timestamp(ds.time.values[0]) time_max = pd.Timestamp(ds.time.values[-1]) total_work = 100 with monitor.starting('LTA', total_work=total_work): monitor.progress(work=0) step = total_work / 12 kwargs = {'monitor': monitor, 'step': step} retset = retset.groupby('time.month', squeeze=False).apply(_mean, **kwargs) # Make the return dataset CF compliant retset = retset.rename({'month': 'time'}) retset['time'] = pd.date_range('{}-01-01'.format(time_min.year), freq='MS', periods=12) climatology_bounds = xr.DataArray(data=np.tile([time_min, time_max], (12, 1)), dims=['time', 'nv'], name='climatology_bounds') retset['climatology_bounds'] = climatology_bounds retset.time.attrs = ds.time.attrs retset.time.attrs['climatology'] = 'climatology_bounds' for var in retset.data_vars: try: retset[var].attrs['cell_methods'] = \ retset[var].attrs['cell_methods'] + ' time: mean over years' except KeyError: retset[var].attrs['cell_methods'] = 'time: mean over years' return retset
def write_csv(obj: DataFrameLike.TYPE, file: FileLike.TYPE, columns: VarNamesLike.TYPE = None, na_rep: str = '', delimiter: str = ',', quotechar: str = None, more_args: DictLike.TYPE = None, monitor: Monitor = Monitor.NONE): """ Write comma-separated values (CSV) to plain text file from a DataFrame or Dataset. :param obj: The object to write as CSV; must be a ``DataFrame`` or a ``Dataset``. :param file: The CSV file path. :param columns: The names of variables that should be converted to columns. If given, coordinate variables are included automatically. :param delimiter: Delimiter to use. :param na_rep: A string representation of a missing value (no-data value). :param quotechar: The character used to denote the start and end of a quoted item. Quoted items can include the delimiter and it will be ignored. :param more_args: Other optional keyword arguments. Please refer to Pandas documentation of ``pandas.to_csv()`` function. :param monitor: optional progress monitor """ if obj is None: raise ValidationError('obj must not be None') columns = VarNamesLike.convert(columns) if isinstance(obj, pd.DataFrame): # The following code is needed, because Pandas treats any kw given in kwargs as being set, even if just None. kwargs = DictLike.convert(more_args) if kwargs is None: kwargs = {} if columns: kwargs.update(columns=columns) if delimiter: kwargs.update(sep=delimiter) if na_rep: kwargs.update(na_rep=na_rep) if quotechar: kwargs.update(quotechar=quotechar) with monitor.starting('Writing to CSV', 1): obj.to_csv(file, index_label='index', **kwargs) monitor.progress(1) elif isinstance(obj, xr.Dataset): var_names = [var_name for var_name in obj.data_vars if columns is None or var_name in columns] dim_names = None data_vars = [] for var_name in var_names: data_var = obj.data_vars[var_name] if dim_names is None: dim_names = data_var.dims elif dim_names != data_var.dims: raise ValidationError('Not all variables have the same dimensions. ' 'Please select variables so that their dimensions are equal.') data_vars.append(data_var) if dim_names is None: raise ValidationError('None of the selected variables has a dimension.') coord_vars = [] for dim_name in dim_names: if dim_name in obj.coords: coord_var = obj.coords[dim_name] else: coord_var = None for data_var in obj.coords.values(): if len(data_var.dims) == 1 and data_var.dims[0] == dim_name: coord_var = data_var break if coord_var is None: raise ValueError(f'No coordinate variable found for dimension "{dim_name}"') coord_vars.append(coord_var) coord_indexes = [range(len(coord_var)) for coord_var in coord_vars] num_coords = len(coord_vars) num_rows = 1 for coord_var in coord_vars: num_rows *= len(coord_var) stream = open(file, 'w') if isinstance(file, str) else file try: # Write header row stream.write('index') for i in range(num_coords): stream.write(delimiter) stream.write(coord_vars[i].name) for data_var in data_vars: stream.write(delimiter) stream.write(data_var.name) stream.write('\n') with monitor.starting('Writing CSV', num_rows): row = 0 for index in itertools.product(*coord_indexes): # Write data row stream.write(str(row)) for i in range(num_coords): coord_value = coord_vars[i].values[index[i]] stream.write(delimiter) stream.write(str(coord_value)) for data_var in data_vars: var_value = data_var.values[index] stream.write(delimiter) stream.write(str(var_value)) stream.write('\n') monitor.progress(1) row += 1 finally: if isinstance(file, str): stream.close() elif obj is None: raise ValidationError('obj must not be None') else: raise ValidationError('obj must be a pandas.DataFrame or a xarray.Dataset')
def plot_line(ds: DatasetLike.TYPE, var_names: VarNamesLike.TYPE, fmt: str = None, label: DimName.TYPE = None, indexers: DictLike.TYPE = None, title: str = None, file: str = None) -> Figure: """ Create a 1D/line plot of variable(s) given by dataset *ds* and variable name(s) *var_names*. :param ds: Dataset or Dataframe that contains the variable(s) named by *var_names*. :param var_names: The name of the variable(s) to plot :param fmt: optional semicolon-separated matplotlib formats, e.g. 1 variable - "b.-" 2 variables - "b.-;r+:" If the number of properties is less than the number of selected variables, the next non-corresponding variable will repeat the first style on the list, and so on. For full reference on matplotlib plot() function, refer to https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html :param file: path to a file in which to save the plot :param label: dimension name to be selected as the x-axis of the plot :param indexers: Optional indexers into data array of *var_names*. The *indexers* is a dictionary or a comma-separated string of key-value pairs that maps the variable's dimension names to constant labels. e.g. "lat=12.4, time='2012-05-02'". :param title: an optional plot title :return: a matplotlib figure object or None if in IPython mode """ ds = DatasetLike.convert(ds) fmt_count = 0 fmt_list = [] if fmt: fmt_list = fmt.split(";") fmt_count = len(fmt_list) if not var_names: raise ValidationError("Missing name for 'vars'") figure = plt.figure() ax = figure.add_subplot(111) figure.subplots_adjust(right=0.65) var_names = VarNamesLike.convert(var_names) if not title: if label: title = ','.join(var_names) + ' over ' + label else: title = ','.join(var_names) if indexers: title = title + '\n' + ' at ' + json.dumps(indexers).strip('"') ax.set_title(title) indexers = DictLike.convert(indexers) ax_var = {} var_count = len(var_names) predefined_fmt = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] if label: ds = get_vars_data(ds, indexers, remaining_dims=[label]) else: ds = get_vars_data(ds, indexers) for i in range(var_count): var_name = var_names[i] var = ds[var_name] if len(var.dims) > 1: raise ValidationError( f'Unable to plot because variable {var_name} has more than one dimension: {var.dims}.' f' To specify value(s) of these dimension(s), please use the indexers.' ) var_label = var_name + ' (' + var.attrs[ 'units'] + ')' if 'units' in var.attrs else var_name properties_dict = {} indexers = DictLike.convert(indexers) if fmt is None: selected_fmt = predefined_fmt[i % len(predefined_fmt)] else: selected_fmt = fmt_list[i % fmt_count] if label: x_axis = var[label] elif 'time' in var: x_axis = var.time else: x_axis = [] # to differentiate the creation of y-axis of the first and the nth variable if i == 0: if len(x_axis) > 0: ax.plot(x_axis, var, selected_fmt, **properties_dict) else: ax.plot(var, selected_fmt, **properties_dict) ax.set_ylabel(var_label, wrap=True) ax.yaxis.label.set_color(selected_fmt[0]) ax.tick_params(axis='y', colors=selected_fmt[0]) else: ax_var[var_name] = ax.twinx() if len(ax_var) > 1: ax_var[var_name].spines["right"].set_position( ("axes", 1 + ((i - 1) * 0.2))) ax_var[var_name].set_frame_on(True) ax_var[var_name].patch.set_visible(False) if len(x_axis) > 0: ax_var[var_name].plot(x_axis, var, selected_fmt, **properties_dict) else: ax_var[var_name].plot(var, selected_fmt, **properties_dict) ax_var[var_name].set_ylabel(var_label, wrap=True) ax_var[var_name].yaxis.label.set_color(selected_fmt[0]) ax_var[var_name].tick_params(axis='y', colors=selected_fmt[0]) ax.tick_params(axis='x', rotation=45) if label in ds and 'long_name' in ds[label].attrs: ax.set_xlabel(ds[label].attrs['long_name']) figure.tight_layout() if file: figure.savefig(file, dpi=600) return figure if not in_notebook() else None
def _make_local(self, local_ds: 'LocalDataSource', time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): local_id = local_ds.id time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) local_path = os.path.join(local_ds.data_store.data_store_path, local_id) data_store_path = local_ds.data_store.data_store_path if not os.path.exists(local_path): os.makedirs(local_path) monitor.start("Sync " + self.id, total_work=len(self._files.items())) for remote_relative_filepath, coverage in self._files.items(): child_monitor = monitor.child(work=1) file_name = os.path.basename(remote_relative_filepath) local_relative_filepath = os.path.join(local_id, file_name) local_absolute_filepath = os.path.join(data_store_path, local_relative_filepath) remote_absolute_filepath = os.path.join( self._data_store.data_store_path, remote_relative_filepath) if isinstance(coverage, Tuple): time_coverage_start = coverage[0] time_coverage_end = coverage[1] if not time_range or time_coverage_start >= time_range[ 0] and time_coverage_end <= time_range[1]: if region or var_names: do_update_of_variables_meta_info_once = True do_update_of_region_meta_info_once = True try: remote_dataset = xr.open_dataset( remote_absolute_filepath) if var_names: remote_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.data_vars.keys() if var_name not in var_names ]) if region: remote_dataset = normalize_impl(remote_dataset) remote_dataset = subset_spatial_impl( remote_dataset, region) geo_lon_min, geo_lat_min, geo_lon_max, geo_lat_max = region.bounds remote_dataset.attrs[ 'geospatial_lat_min'] = geo_lat_min remote_dataset.attrs[ 'geospatial_lat_max'] = geo_lat_max remote_dataset.attrs[ 'geospatial_lon_min'] = geo_lon_min remote_dataset.attrs[ 'geospatial_lon_max'] = geo_lon_max if do_update_of_region_meta_info_once: local_ds.meta_info[ 'bbox_maxx'] = geo_lon_max local_ds.meta_info[ 'bbox_minx'] = geo_lon_min local_ds.meta_info[ 'bbox_maxy'] = geo_lat_max local_ds.meta_info[ 'bbox_miny'] = geo_lat_min do_update_of_region_meta_info_once = False if compression_enabled: for sel_var_name in remote_dataset.variables.keys( ): remote_dataset.variables.get( sel_var_name).encoding.update( encoding_update) remote_dataset.to_netcdf(local_absolute_filepath) child_monitor.progress( work=1, msg=str(time_coverage_start)) finally: if do_update_of_variables_meta_info_once: variables_info = local_ds.meta_info.get( 'variables', []) local_ds.meta_info['variables'] = [ var_info for var_info in variables_info if var_info.get('name') in remote_dataset. variables.keys() and var_info.get('name') not in remote_dataset.dims.keys() ] do_update_of_variables_meta_info_once = False local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) child_monitor.done() else: shutil.copy(remote_absolute_filepath, local_absolute_filepath) local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() monitor.done() return local_id
def detect_outliers(ds: xr.Dataset, var: VarNamesLike.TYPE, threshold_low: float = 0.05, threshold_high: float = 0.95, quantiles: bool = True, mask: bool = False, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Detect outliers in the given Dataset. When mask=True the input dataset should not contain nan values, otherwise all existing nan values will be marked as 'outliers' in the mask data array added to the output dataset. :param ds: The dataset or dataframe for which to do outlier detection :param var: Variable or variables in the dataset to which to do outlier detection. Note that when multiple variables are selected, absolute threshold values might not make much sense. Wild cards can be used to select multiple variables matching a pattern. :param threshold_low: Values less or equal to this will be removed/masked :param threshold_high: Values greater or equal to this will be removed/masked :param quantiles: If True, threshold values are treated as quantiles, otherwise as absolute values. :param mask: If True, an ancillary variable containing flag values for outliers will be added to the dataset. Otherwise, outliers will be replaced with nan directly in the data variables. :param monitor: A progress monitor. :return: The dataset with outliers masked or replaced with nan """ ds = DatasetLike.convert(ds) # Create a list of variable names on which to perform outlier detection # based on the input comma separated list that can contain wildcards var_patterns = VarNamesLike.convert(var) all_vars = list(ds.data_vars.keys()) variables = list() for pattern in var_patterns: leave = fnmatch.filter(all_vars, pattern) variables = variables + leave # For each array in the dataset for which we should detect outliers, detect # outliers ret_ds = ds.copy() with monitor.starting("detect_outliers", total_work=len(variables) * 3): for var_name in variables: if quantiles: # Get threshold values with monitor.child(1).observing("quantile low"): threshold_low = ret_ds[var_name].quantile(threshold_low) with monitor.child(1).observing("quantile high"): threshold_high = ret_ds[var_name].quantile(threshold_high) else: monitor.progress(2) # If not mask, put nans in the data arrays for min/max outliers if not mask: arr = ret_ds[var_name] attrs = arr.attrs ret_ds[var_name] = arr.where((arr > threshold_low) & (arr < threshold_high)) ret_ds[var_name].attrs = attrs else: # Create and add a data variable containing the mask for this data # variable _mask_outliers(ret_ds, var_name, threshold_low, threshold_high) monitor.progress(1) return ret_ds
def data_frame_aggregate(df: DataFrameLike.TYPE, var_names: VarNamesLike.TYPE = None, aggregate_geometry: bool = False, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ Aggregate columns into count, mean, median, sum, std, min, and max. Return a new (Geo)DataFrame with a single row containing all aggregated values. Specify whether the geometries of the GeoDataFrame are to be aggregated. All geometries are merged union-like. The return data type will always be the same as the input data type. :param df: The (Geo)DataFrame to be analysed :param var_names: Variables to be aggregated ('None' uses all aggregatable columns) :param aggregate_geometry: Aggregate (union like) the geometry and add it to the resulting GeoDataFrame :param monitor: Monitor for progress bar :return: returns either DataFrame or GeoDataFrame. Keeps input data type """ vns = VarNamesLike.convert(var_names) df_is_geo = isinstance(df, gpd.GeoDataFrame) aggregations = ["count", "mean", "median", "sum", "std", "min", "max"] # Check var names integrity (aggregatable, exists in data frame) types_accepted_for_agg = ['float64', 'int64', 'bool'] agg_columns = list(df.select_dtypes(include=types_accepted_for_agg).columns) if df_is_geo: agg_columns.append('geometry') columns = list(df.columns) if vns is None: vns = agg_columns diff = list(set(vns) - set(columns)) if len(diff) > 0: raise ValidationError('Variable ' + ','.join(diff) + ' not in data frame!') diff = list(set(vns) - set(agg_columns)) if len(diff) > 0: raise ValidationError('Variable(s) ' + ','.join(diff) + ' not aggregatable!') try: df['geometry'] except KeyError as e: raise ValidationError('Variable geometry not in GEO data frame!') from e # Aggregate columns if vns is None: df_buff = df.select_dtypes(include=types_accepted_for_agg).agg(aggregations) else: df_buff = df[vns].select_dtypes(include=types_accepted_for_agg).agg(aggregations) res = {} for n in df_buff.columns: for a in aggregations: val = df_buff[n][a] h = n + '_' + a res[h] = [val] df_agg = pd.DataFrame(res) # Aggregate (union) geometry if GeoDataFrame if df_is_geo and aggregate_geometry: total_work = 100 num_work_rows = 1 + len(df) // total_work with monitor.starting('Aggregating geometry: ', total_work): multi_polygon = shapely.geometry.MultiPolygon() i = 0 for rec in df.geometry: if monitor.is_cancelled(): break # noinspection PyBroadException try: multi_polygon = multi_polygon.union(other=rec) except Exception: pass if i % num_work_rows == 0: monitor.progress(work=1) i += 1 df_agg = gpd.GeoDataFrame(df_agg, geometry=[multi_polygon], crs=df.crs) return df_agg
def test_make_local_and_update(self): soilmoisture_data_sources = self.data_store.query( query_expr='esacci.SOILMOISTURE.day.L3S.SSMV.multi-sensor.multi-platform.COMBINED.02-1.r1') soilmoisture_data_source = soilmoisture_data_sources[0] reference_path = os.path.join(os.path.dirname(__file__), os.path.normpath('resources/datasources/local/files/')) def find_files_mock(_, time_range): def build_file_item(item_name: str, date_from: datetime, date_to: datetime, size: int): return [item_name, date_from, date_to, size, {'OPENDAP': os.path.join(reference_path, item_name), 'HTTPServer': 'file:' + urllib.request.pathname2url(os.path.join(reference_path, item_name))}] reference_files = { 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781114000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 14, 0, 0), 'date_to': datetime.datetime(1978, 11, 14, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781115000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 15, 0, 0), 'date_to': datetime.datetime(1978, 11, 15, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781116000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 16, 0, 0), 'date_to': datetime.datetime(1978, 11, 16, 23, 59), 'size': 21511378 } } reference_files_list = [] for reference_file in reference_files.items(): file_name = reference_file[0] file_date_from = reference_file[1].get('date_from') file_date_to = reference_file[1].get('date_to') file_size = reference_file[1].get('size') if time_range: if file_date_from >= time_range[0] and file_date_to <= time_range[1]: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) else: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) return reference_files_list with unittest.mock.patch('cate.ds.esa_cci_odp.EsaCciOdpDataSource._find_files', find_files_mock): with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'local_ds_test' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) try: new_ds = soilmoisture_data_source.make_local(new_ds_title, time_range=new_ds_time_range) except Exception: raise ValueError(reference_path, os.listdir(reference_path)) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), new_ds_time_range) new_ds_w_one_variable_title = 'local_ds_test_var' new_ds_w_one_variable_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_one_variable_var_names = VarNamesLike.convert(['sm']) new_ds_w_one_variable = soilmoisture_data_source.make_local( new_ds_w_one_variable_title, time_range=new_ds_w_one_variable_time_range, var_names=new_ds_w_one_variable_var_names ) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_w_one_variable_title) ds = new_ds_w_one_variable.open_dataset() new_ds_w_one_variable_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(ds.variables), set(new_ds_w_one_variable_var_names)) new_ds_w_region_title = 'from_local_to_local_region' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) new_ds_w_region_title = 'from_local_to_local_region_one_var' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) new_ds_w_region_title = 'from_local_to_local_region_two_var_sm_uncertainty' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm', 'sm_uncertainty']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) empty_ds_timerange = (datetime.datetime(2017, 12, 1, 0, 0), datetime.datetime(2017, 12, 31, 23, 59)) with self.assertRaises(DataAccessError) as cm: soilmoisture_data_source.make_local('empty_ds', time_range=empty_ds_timerange) self.assertEqual(f'Data source "{soilmoisture_data_source.id}" does not' f' seem to have any datasets in given' f' time range {TimeRangeLike.format(empty_ds_timerange)}', str(cm.exception)) new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 14, 23, 59))) new_ds = soilmoisture_data_source.make_local("title_test_copy", time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], soilmoisture_data_source.meta_info['title']) title = "Title Test!" new_ds = soilmoisture_data_source.make_local("title_test_set", title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], title)
def plot_line(ds: DatasetLike.TYPE, var_names: VarNamesLike.TYPE, fmt: str = None, label: DimName.TYPE = None, indexers: DictLike.TYPE = None, title: str = None, file: str = None) -> Figure: """ Create a 1D/line plot of variable(s) given by dataset *ds* and variable name(s) *var_names*. :param ds: Dataset or Dataframe that contains the variable(s) named by *var_names*. :param var_names: The name of the variable(s) to plot :param fmt: optional semicolon-separated matplotlib formats, e.g. 1 variable - "b.-" 2 variables - "b.-;r+:" If the number of properties is less than the number of selected variables, the next non-corresponding variable will repeat the first style on the list, and so on. For full reference on matplotlib plot() function, refer to https://matplotlib.org/api/_as_gen/matplotlib.pyplot.plot.html :param file: path to a file in which to save the plot :param label: dimension name to be selected as the x-axis of the plot :param indexers: Optional indexers into data array of *var_names*. The *indexers* is a dictionary or a comma-separated string of key-value pairs that maps the variable's dimension names to constant labels. e.g. "lat=12.4, time='2012-05-02'". :param title: an optional plot title :return: a matplotlib figure object or None if in IPython mode """ ds = DatasetLike.convert(ds) fmt_count = 0 fmt_list = [] if fmt: fmt_list = fmt.split(";") fmt_count = len(fmt_list) if not var_names: raise ValidationError("Missing name for 'vars'") figure = plt.figure() ax = figure.add_subplot(111) figure.subplots_adjust(right=0.65) var_names = VarNamesLike.convert(var_names) if not title: if label: title = ','.join(var_names) + ' over ' + label else: title = ','.join(var_names) if indexers: title = title + '\n' + ' at ' + json.dumps(indexers).strip('"') ax.set_title(title) indexers = DictLike.convert(indexers) ax_var = {} var_count = len(var_names) predefined_fmt = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] if label: ds = get_vars_data(ds, indexers, remaining_dims=[label]) else: ds = get_vars_data(ds, indexers) for i in range(var_count): var_name = var_names[i] var = ds[var_name] if len(var.dims) > 1: raise ValidationError(f'Unable to plot because variable {var_name} has more than one dimension: {var.dims}.' f' To specify value(s) of these dimension(s), please use the indexers.') var_label = var_name + ' (' + var.attrs['units'] + ')' if 'units' in var.attrs else var_name properties_dict = {} indexers = DictLike.convert(indexers) if fmt is None: selected_fmt = predefined_fmt[i % len(predefined_fmt)] else: selected_fmt = fmt_list[i % fmt_count] if label: x_axis = var[label] elif 'time' in var: x_axis = var.time else: x_axis = [] # to differentiate the creation of y-axis of the first and the nth variable if i == 0: if len(x_axis) > 0: ax.plot(x_axis, var, selected_fmt, **properties_dict) else: ax.plot(var, selected_fmt, **properties_dict) ax.set_ylabel(var_label, wrap=True) ax.yaxis.label.set_color(selected_fmt[0]) ax.tick_params(axis='y', colors=selected_fmt[0]) else: ax_var[var_name] = ax.twinx() if len(ax_var) > 1: ax_var[var_name].spines["right"].set_position(("axes", 1 + ((i - 1) * 0.2))) ax_var[var_name].set_frame_on(True) ax_var[var_name].patch.set_visible(False) if len(x_axis) > 0: ax_var[var_name].plot(x_axis, var, selected_fmt, **properties_dict) else: ax_var[var_name].plot(var, selected_fmt, **properties_dict) ax_var[var_name].set_ylabel(var_label, wrap=True) ax_var[var_name].yaxis.label.set_color(selected_fmt[0]) ax_var[var_name].tick_params(axis='y', colors=selected_fmt[0]) ax.tick_params(axis='x', rotation=45) if label in ds and 'long_name' in ds[label].attrs: ax.set_xlabel(ds[label].attrs['long_name']) figure.tight_layout() if file: figure.savefig(file, dpi=600) return figure if not in_notebook() else None
def _make_local(self, local_ds: 'LocalDataSource', time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): # local_name = local_ds.name local_id = local_ds.name time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) local_path = os.path.join(local_ds.data_store.data_store_path, local_id) data_store_path = local_ds.data_store.data_store_path if not os.path.exists(local_path): os.makedirs(local_path) monitor.start("Sync " + self.name, total_work=len(self._files.items())) for remote_relative_filepath, coverage in self._files.items(): child_monitor = monitor.child(work=1) file_name = os.path.basename(remote_relative_filepath) local_relative_filepath = os.path.join(local_id, file_name) local_absolute_filepath = os.path.join(data_store_path, local_relative_filepath) remote_absolute_filepath = os.path.join( self._data_store.data_store_path, remote_relative_filepath) if isinstance(coverage, Tuple): time_coverage_start = coverage[0] time_coverage_end = coverage[1] remote_netcdf = None local_netcdf = None if not time_range or time_coverage_start >= time_range[ 0] and time_coverage_end <= time_range[1]: if region or var_names: try: remote_netcdf = NetCDF4DataStore( remote_absolute_filepath) local_netcdf = NetCDF4DataStore( local_absolute_filepath, mode='w', persist=True) local_netcdf.set_attributes( remote_netcdf.get_attrs()) remote_dataset = xr.Dataset.load_store( remote_netcdf) process_region = False if region: geo_lat_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_min') geo_lat_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_max') geo_lon_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_min') geo_lon_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_max') geo_lat_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_resolution') geo_lon_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_resolution') if not (isnan(geo_lat_min) or isnan(geo_lat_max) or isnan(geo_lon_min) or isnan(geo_lon_max) or isnan(geo_lat_res) or isnan(geo_lon_res)): process_region = True [lat_min, lon_min, lat_max, lon_max] = region.bounds lat_min = floor( (lat_min - geo_lat_min) / geo_lat_res) lat_max = ceil( (lat_max - geo_lat_min) / geo_lat_res) lon_min = floor( (lon_min - geo_lon_min) / geo_lon_res) lon_max = ceil( (lon_max - geo_lon_min) / geo_lon_res) # TODO (kbernat): check why dataset.sel fails! remote_dataset = remote_dataset.isel( drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) geo_lat_max = lat_max * geo_lat_res + geo_lat_min geo_lat_min += lat_min * geo_lat_res geo_lon_max = lon_max * geo_lon_res + geo_lon_min geo_lon_min += lon_min * geo_lon_res if not var_names: var_names = [ var_name for var_name in remote_netcdf.variables.keys() ] var_names.extend([ coord_name for coord_name in remote_dataset.coords.keys() if coord_name not in var_names ]) child_monitor.start(label=file_name, total_work=len(var_names)) for sel_var_name in var_names: var_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.variables.keys() if var_name != sel_var_name ]) if compression_enabled: var_dataset.variables.get( sel_var_name).encoding.update( encoding_update) local_netcdf.store_dataset(var_dataset) child_monitor.progress(work=1, msg=sel_var_name) if process_region: local_netcdf.set_attribute( 'geospatial_lat_min', geo_lat_min) local_netcdf.set_attribute( 'geospatial_lat_max', geo_lat_max) local_netcdf.set_attribute( 'geospatial_lon_min', geo_lon_min) local_netcdf.set_attribute( 'geospatial_lon_max', geo_lon_max) finally: if remote_netcdf: remote_netcdf.close() if local_netcdf: local_netcdf.close() local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() else: shutil.copy(remote_absolute_filepath, local_absolute_filepath) local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() monitor.done() return local_id
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None, monitor: Monitor = Monitor.NONE) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None var_names = VarNamesLike.convert(var_names) if var_names else None paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance( time_series[i], datetime ) and time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if not paths: raise self._empty_error(time_range) paths = sorted(set(paths)) try: excluded_variables = self._meta_info.get('exclude_variables') if excluded_variables: drop_variables = [ variable.get('name') for variable in excluded_variables ] else: drop_variables = None # TODO: combine var_names and drop_variables return open_xarray_dataset(paths, region=region, var_names=var_names, drop_variables=drop_variables, monitor=monitor) except HTTPError as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e) from e except (URLError, socket.timeout) as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e, error_cls=NetworkError) from e except OSError as e: raise self._cannot_access_error(time_range, region, var_names, verb="open", cause=e) from e
def test_make_local_and_update(self): soilmoisture_data_sources = self.data_store.query( query_expr='esacci.SOILMOISTURE.day.L3S.SSMV.multi-sensor.multi-platform.COMBINED.02-1.r1') soilmoisture_data_source = soilmoisture_data_sources[0] reference_path = os.path.join(os.path.dirname(__file__), os.path.normpath('resources/datasources/local/files/')) def find_files_mock(_, time_range): def build_file_item(item_name: str, date_from: datetime, date_to: datetime, size: int): return [item_name, date_from, date_to, size, {'OPENDAP': os.path.join(reference_path, item_name), 'HTTPServer': 'file:' + urllib.request.pathname2url(os.path.join(reference_path, item_name))}] reference_files = { 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781114000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 14, 0, 0), 'date_to': datetime.datetime(1978, 11, 14, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781115000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 15, 0, 0), 'date_to': datetime.datetime(1978, 11, 15, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781116000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 16, 0, 0), 'date_to': datetime.datetime(1978, 11, 16, 23, 59), 'size': 21511378 } } reference_files_list = [] for reference_file in reference_files.items(): file_name = reference_file[0] file_date_from = reference_file[1].get('date_from') file_date_to = reference_file[1].get('date_to') file_size = reference_file[1].get('size') if time_range: if file_date_from >= time_range[0] and file_date_to <= time_range[1]: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) else: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) return reference_files_list with unittest.mock.patch('cate.ds.esa_cci_odp.EsaCciOdpDataSource._find_files', find_files_mock): with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'local_ds_test' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) try: new_ds = soilmoisture_data_source.make_local(new_ds_title, time_range=new_ds_time_range) except Exception: raise ValueError(reference_path, os.listdir(reference_path)) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), new_ds_time_range) new_ds_w_one_variable_title = 'local_ds_test_var' new_ds_w_one_variable_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_one_variable_var_names = VarNamesLike.convert(['sm']) new_ds_w_one_variable = soilmoisture_data_source.make_local( new_ds_w_one_variable_title, time_range=new_ds_w_one_variable_time_range, var_names=new_ds_w_one_variable_var_names ) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_w_one_variable_title) ds = new_ds_w_one_variable.open_dataset() new_ds_w_one_variable_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(ds.variables), set(new_ds_w_one_variable_var_names)) new_ds_w_region_title = 'from_local_to_local_region' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) new_ds_w_region_title = 'from_local_to_local_region_one_var' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) new_ds_w_region_title = 'from_local_to_local_region_two_var_sm_uncertainty' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm', 'sm_uncertainty']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) empty_ds_timerange = (datetime.datetime(2017, 12, 1, 0, 0), datetime.datetime(2017, 12, 31, 23, 59)) with self.assertRaises(DataAccessError) as cm: soilmoisture_data_source.make_local('empty_ds', time_range=empty_ds_timerange) self.assertEqual(f'Data source "{soilmoisture_data_source.id}" does not' f' seem to have any datasets in given' f' time range {TimeRangeLike.format(empty_ds_timerange)}', str(cm.exception)) new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 14, 23, 59))) new_ds = soilmoisture_data_source.make_local("title_test_copy", time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], soilmoisture_data_source.meta_info['title']) title = "Title Test!" new_ds = soilmoisture_data_source.make_local("title_test_set", title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], title)