def test_convert(self): coords = [(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0)] expected = Polygon(coords) actual = PolygonLike.convert(coords) self.assertTrue(actual, expected) with self.assertRaises(ValueError) as err: PolygonLike.convert('aaa') self.assertEqual('cannot convert geometry to a valid Polygon: aaa', str(err.exception)) self.assertEqual(None, PolygonLike.convert(None))
def test_load_old_datasource_from_json_dict(self): test_data = { 'name': 'local.test_name', 'meta_data': { 'temporal_coverage': "2001-01-01 00:00:00,2001-01-31 23:59:59", 'spatial_coverage': "-180,-90,180,90", 'variables': ['var_test_1', 'var_test_2'], }, "meta_info": {}, 'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'], ['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']] } data_source = LocalDataSource.from_json_dict( json_dict=test_data, data_store=self.data_store) self.assertIsNotNone(data_source) self.assertEqual( data_source.temporal_coverage(), TimeRangeLike.convert( test_data.get('meta_data').get('temporal_coverage'))) self.assertEqual( data_source.spatial_coverage(), PolygonLike.convert( test_data.get('meta_data').get('spatial_coverage'))) self.assertListEqual( [var.get('name') for var in data_source.variables_info], test_data.get('meta_data').get('variables')) test_data = { 'name': 'local.test_name', 'meta_data': { 'temporal_covrage': "2001-01-01 00:00:00,2001-01-31 23:59:59", 'spatial_coverage': "-180,-90,180,90", 'variables': ['var_test_1', 'var_test_2'], }, "meta_info": {}, 'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'], ['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']] } data_source = LocalDataSource.from_json_dict( json_dict=test_data, data_store=self.data_store) self.assertIsNotNone(data_source) self.assertEqual( data_source.temporal_coverage(), TimeRangeLike.convert( test_data.get('meta_data').get('temporal_covrage'))) self.assertEqual( data_source.spatial_coverage(), PolygonLike.convert( test_data.get('meta_data').get('spatial_coverage'))) self.assertListEqual( [var.get('name') for var in data_source.variables_info], test_data.get('meta_data').get('variables'))
def test_make_local(self): data_source = self._local_data_store.query('local_w_temporal')[0] with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'from_local_to_local' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds = data_source.make_local(new_ds_title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), TimeRangeLike.convert( (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)))) new_ds_2_title = 'from_local_to_local_var' new_ds_2_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds_2_vars = VarNamesLike.convert(['sm']) new_ds_w_one_variable = data_source.make_local(new_ds_2_title, time_range=new_ds_2_time_range, var_names=new_ds_2_vars) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_2_title) data_set = new_ds_w_one_variable.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'}) new_ds_3_title = 'from_local_to_local_range' new_ds_3_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) new_ds_3_vars = VarNamesLike.convert(['sm']) new_ds_3_region = PolygonLike.convert("10,10,20,20") new_ds_w_region = data_source.make_local(new_ds_3_title, time_range=new_ds_3_time_range, var_names=new_ds_3_vars, region=new_ds_3_region) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_3_title) self.assertEqual(new_ds_w_region.spatial_coverage(), PolygonLike.convert("10,10,20,20")) data_set = new_ds_w_region.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'}) no_data = data_source.make_local('no_data', time_range=(datetime.datetime(2020, 11, 14, 0, 0), datetime.datetime(2020, 11, 15, 23, 59))) self.assertIsNone(no_data)
def data_frame_subset(gdf: gpd.GeoDataFrame, region_op: bool = 'intersects', region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None) -> gpd.GeoDataFrame: """ Create a GeoDataFrame subset from given variables (data frame columns) and/or region. :param gdf: A GeoDataFrame. :param region_op: The geometric operation to be performed if *region* is given. :param region: A region polygon used to filter rows. :param var_names: The variables (columns) to select. :return: A GeoDataFrame subset. """ region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) if not var_names and not region: return gdf if var_names: if 'geometry' not in var_names: var_names = ['geometry'] + var_names gdf = gdf[var_names] if region and region_op: geom_str = PolygonLike.format(region) gdf = data_frame_query(gdf, f'@{region_op}("{geom_str}")') return gdf
def data_frame_subset(gdf: gpd.GeoDataFrame, region_op: bool = 'intersects', region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None) -> gpd.GeoDataFrame: """ Create a GeoDataFrame subset from given variables (data frame columns) and/or region. :param gdf: A GeoDataFrame. :param region_op: The geometric operation to be performed if *region* is given. :param region: A region polygon used to filter rows. :param var_names: The variables (columns) to select. :return: A GeoDataFrame subset. """ region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) if not var_names and not region: return gdf if var_names: if 'geometry' not in var_names: var_names = ['geometry'] + var_names gdf = gdf[var_names] if region and region_op: geom_str = PolygonLike.format(region) gdf = data_frame_query(gdf, f'@{region_op}("{geom_str}")') return gdf
def anomaly_internal(ds: xr.Dataset, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Calculate anomaly using as reference data the mean of an optional region and time slice from the given dataset. If no time slice/spatial region is given, the operation will calculate anomaly using the mean of the whole dataset as the reference. This is done for each data array in the dataset. :param ds: The dataset to calculate anomalies from :param time_range: Time range to use for reference data :param region: Spatial region to use for reference data :param monitor: a progress monitor. :return: The anomaly dataset """ ref = ds.copy() if time_range: time_range = TimeRangeLike.convert(time_range) ref = subset_temporal(ref, time_range) if region: region = PolygonLike.convert(region) ref = subset_spatial(ref, region) with monitor.observing("Calculating anomaly"): ref = ref.mean(keep_attrs=True, skipna=True) diff = ds - ref return diff
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'Open Data Portal\'s data source \'{}\' does not seem to have any data sets'.format(self.id) if time_range is not None: msg += ' in given time range {}'.format(TimeRangeLike.format(time_range)) raise DataAccessError(None, msg) files = self._get_urls_list(selected_file_list, _ODP_PROTOCOL_OPENDAP) try: ds = open_xarray_dataset(files) if region: ds = subset_spatial_impl(ds, region) if var_names: ds = ds.drop([var_name for var_name in ds.data_vars.keys() if var_name not in var_names]) return ds except OSError as e: if time_range: raise DataAccessError(self, "Cannot open remote dataset for time range: {}\n" "Error details: {}" .format(TimeRangeLike.format(time_range), e)) else: raise DataAccessError(self, "Cannot open remote dataset\n" "Error details: {}" .format(TimeRangeLike.format(time_range), e))
def anomaly_internal(ds: xr.Dataset, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> xr.Dataset: """ Calculate anomaly using as reference data the mean of an optional region and time slice from the given dataset. If no time slice/spatial region is given, the operation will calculate anomaly using the mean of the whole dataset as the reference. This is done for each data array in the dataset. :param ds: The dataset to calculate anomalies from :param time_range: Time range to use for reference data :param region: Spatial region to use for reference data :param monitor: a progress monitor. :return: The anomaly dataset """ ref = ds.copy() if time_range: time_range = TimeRangeLike.convert(time_range) ref = subset_temporal(ref, time_range) if region: region = PolygonLike.convert(region) ref = subset_spatial(ref, region) with monitor.observing("Calculating anomaly"): ref = ref.mean(keep_attrs=True, skipna=True) diff = ds - ref return diff
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None if region: region = PolygonLike.convert(region) if var_names: var_names = VarNamesLike.convert(var_names) paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance( time_series[i], datetime ) and time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if paths: paths = sorted(set(paths)) try: ds = open_xarray_dataset(paths) if region: ds = normalize_impl(ds) ds = subset_spatial_impl(ds, region) if var_names: ds = ds.drop([ var_name for var_name in ds.data_vars.keys() if var_name not in var_names ]) return ds except OSError as e: if time_range: raise DataAccessError( "Cannot open local dataset for time range {}:\n" "{}".format(TimeRangeLike.format(time_range), e), source=self) from e else: raise DataAccessError("Cannot open local dataset:\n" "{}".format(e), source=self) from e else: if time_range: raise DataAccessError( "No local datasets available for\nspecified time range {}". format(TimeRangeLike.format(time_range)), source=self) else: raise DataAccessError("No local datasets available", source=self)
def test_format(self): self.assertEqual(PolygonLike.format(None), '') coords = [(10.4, 20.2), (30.8, 20.2), (30.8, 40.8), (10.4, 40.8)] pol = PolygonLike.convert(coords) self.assertEqual( PolygonLike.format(pol), 'POLYGON ((10.4 20.2, 30.8 20.2, 30.8 40.8, 10.4 40.8, 10.4 20.2))' )
def spatial_coverage(self): if not self._spatial_coverage and \ set(self._meta_info.keys()).issuperset({'bbox_minx', 'bbox_miny', 'bbox_maxx', 'bbox_maxy'}): self._spatial_coverage = PolygonLike.convert(",".join([ self._meta_info.get('bbox_minx'), self._meta_info.get('bbox_miny'), self._meta_info.get('bbox_maxx'), self._meta_info.get('bbox_maxy')]) ) return self._spatial_coverage
def spatial_coverage(self): if not self._spatial_coverage and \ set(self._meta_info.keys()).issuperset({'bbox_minx', 'bbox_miny', 'bbox_maxx', 'bbox_maxy'}): self._spatial_coverage = PolygonLike.convert(",".join([ self._meta_info.get('bbox_minx'), self._meta_info.get('bbox_miny'), self._meta_info.get('bbox_maxx'), self._meta_info.get('bbox_maxy') ])) return self._spatial_coverage
def test_make_local(self): data_source = self._local_data_store.query('local_w_temporal')[0] with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds = data_source.make_local( 'from_local_to_local', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) self.assertEqual(new_ds.name, 'local.from_local_to_local') self.assertEqual( new_ds.temporal_coverage(), TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)))) data_source.update_local(new_ds.name, (datetime.datetime(1978, 11, 15, 00, 00), datetime.datetime(1978, 11, 16, 23, 59))) self.assertEqual( new_ds.temporal_coverage(), TimeRangeLike.convert((datetime.datetime(1978, 11, 15, 0, 0), datetime.datetime(1978, 11, 16, 23, 59)))) with self.assertRaises(ValueError) as context: data_source.update_local( "wrong_ds_name", (datetime.datetime(1978, 11, 15, 00, 00), datetime.datetime(1978, 11, 16, 23, 59))) self.assertTrue("Couldn't find local DataSource", context.exception.args[0]) new_ds_w_one_variable = data_source.make_local( 'from_local_to_local_var', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)), None, ['sm']) self.assertEqual(new_ds_w_one_variable.name, 'local.from_local_to_local_var') data_set = new_ds_w_one_variable.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'}) new_ds_w_region = data_source.make_local( 'from_local_to_local_region', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)), "10,10,20,20", ['sm']) # type: LocalDataSource self.assertEqual(new_ds_w_region.name, 'local.from_local_to_local_region') self.assertEqual(new_ds_w_region.spatial_coverage(), PolygonLike.convert("10,10,20,20")) data_set = new_ds_w_region.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'})
def spatial_coverage(self) -> Optional[PolygonLike]: if self._catalogue_data \ and self._catalogue_data.get('bbox_minx', None) and self._catalogue_data.get('bbox_miny', None) \ and self._catalogue_data.get('bbox_maxx', None) and self._catalogue_data.get('bbox_maxy', None): return PolygonLike.convert([ self._catalogue_data.get('bbox_minx'), self._catalogue_data.get('bbox_miny'), self._catalogue_data.get('bbox_maxx'), self._catalogue_data.get('bbox_maxy') ]) return None
def make_local(self, local_name: str, local_id: str = None, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> Optional[DataSource]: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: add_to_data_store_registry() local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: raise ValueError('Cannot initialize `local` DataStore') _uuid = LocalDataStore.generate_uuid(ref_id=self.id, time_range=time_range, region=region, var_names=var_names) if not local_name or len(local_name) == 0: local_name = "local.{}.{}".format(self.id, _uuid) existing_ds_list = local_store.query(ds_id=local_name) if len(existing_ds_list) == 1: return existing_ds_list[0] else: existing_ds_list = local_store.query(ds_id='local.%s' % local_name) if len(existing_ds_list) == 1: if existing_ds_list[0].meta_info.get('uuid', None) == _uuid: return existing_ds_list[0] else: raise ValueError('Datastore {} already contains dataset {}'.format(local_store.id, local_name)) local_meta_info = self.meta_info.copy() local_meta_info['ref_uuid'] = local_meta_info.get('uuid', None) local_meta_info['uuid'] = _uuid local_ds = local_store.create_data_source(local_name, region, local_name, time_range=time_range, var_names=var_names, meta_info=self.meta_info.copy()) if local_ds: if not local_ds.is_complete: self._make_local(local_ds, time_range, region, var_names, monitor=monitor) if local_ds.is_empty: local_store.remove_data_source(local_ds) return None local_store.register_ds(local_ds) return local_ds return None
def subset_spatial(ds: xr.Dataset, region: PolygonLike.TYPE, mask: bool = True) -> xr.Dataset: """ Do a spatial subset of the dataset :param ds: Dataset to subset :param region: Spatial region to subset :param mask: Should values falling in the bounding box of the polygon but not the polygon itself be masked with NaN. :return: Subset dataset """ region = PolygonLike.convert(region) return adjust_spatial_attrs(subset_spatial_impl(ds, region, mask))
def test_load_datasource_from_json_dict(self): test_data = { 'name': 'local.test_name2', "meta_info": { "temporal_coverage_start": "2001-01-01T00:00:00", "temporal_coverage_end": "2001-01-31T23:59:59", "bbox_maxx": "180.0", "bbox_maxy": "90.0", "bbox_minx": "-180.0", "bbox_miny": "-90.0", "variables": [{ "name": "var_1", "units": "kelvin", "long_name": "var_1 long name..", "standard_name": "std_var_1" }, { "name": "var_2", "units": "celsius", "long_name": "var_2 long name..", "standard_name": "std_var_2" }] }, 'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'], ['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']] } data_source = LocalDataSource.from_json_dict( json_dict=test_data, data_store=self.data_store) self.assertIsNotNone(data_source) self.assertEqual( data_source.temporal_coverage(), TimeRangeLike.convert("{},{}".format( test_data.get('meta_info').get('temporal_coverage_start'), test_data.get('meta_info').get('temporal_coverage_end')))) self.assertEqual( data_source.spatial_coverage(), PolygonLike.convert(",".join([ test_data.get('meta_info').get('bbox_minx'), test_data.get('meta_info').get('bbox_miny'), test_data.get('meta_info').get('bbox_maxx'), test_data.get('meta_info').get('bbox_maxy'), ]))) self.assertListEqual(data_source.variables_info, test_data.get('meta_info').get('variables'))
def _crosses_antimeridian(region: PolygonLike.TYPE) -> bool: """ Determine if the given region crosses the Antimeridian line, by converting the given Polygon from -180;180 to 0;360 and checking if the antimeridian line crosses it. This only works with Polygons without holes :param region: PolygonLike to test """ region = PolygonLike.convert(region) # Retrieving the points of the Polygon are a bit troublesome, parsing WKT # is more straightforward and probably faster new_wkt = 'POLYGON ((' # [10:-2] gets rid of POLYGON (( and )) for point in dumps(region)[10:-2].split(','): point = point.strip() lon, lat = point.split(' ') lon = float(lon) if -180 <= lon < 0: lon += 360 new_wkt += '{} {}, '.format(lon, lat) new_wkt = new_wkt[:-2] + '))' converted = loads(new_wkt) # There's a problem at this point. Any polygon crossed by the zeroth # meridian can in principle convert to an inverted polygon that is crossed # by the antimeridian. if not converted.is_valid: # The polygon 'became' invalid upon conversion => probably the original # polygon is what we want return False test_line = LineString([(180, -90), (180, 90)]) if test_line.crosses(converted): # The converted polygon seems to be valid and crossed by the # antimeridian. At this point there's no 'perfect' way how to tell if # we wanted the converted polygon or the original one. # A simple heuristic is to check for size. The smaller one is quite # likely the desired one if converted.area < region.area: return True else: return False
def __init__(self, ds_id: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, meta_info: dict = None, status: DataSourceStatus = None): self._id = ds_id if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert( temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert( tuple([ files_range[0][0], files_range[files_number - 1][1] ])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert( (files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert( spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert(variables) if variables else [] self._meta_info = meta_info if meta_info else OrderedDict() if self._variables and not self._meta_info.get('variables', None): self._meta_info['variables'] = [{ 'name': var_name, 'units': '', 'long_name': '', 'standard_name': '' } for var_name in self._variables] self._status = status if status else DataSourceStatus.READY
def _generic_index_calculation( ds: xr.Dataset, var: VarName.TYPE, region: PolygonLike.TYPE, window: int, file: str, name: str, threshold: float = None, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ A generic index calculation. Where an index is defined as an anomaly against the given reference of a moving average of the given window size of the given given region of the given variable of the given dataset. :param ds: Dataset from which to calculate the index :param var: Variable from which to calculate index :param region: Spatial subset from which to calculate the index :param window: Window size for the moving average :param file: Path to the reference file :param threshold: Absolute threshold that indicates an ENSO event :param name: Name of the index :param monitor: a progress monitor. :return: A dataset that contains the index timeseries """ var = VarName.convert(var) region = PolygonLike.convert(region) with monitor.starting("Calculate the index", total_work=2): ds = select_var(ds, var) ds_subset = subset_spatial(ds, region) anom = anomaly_external(ds_subset, file, monitor=monitor.child(1)) with monitor.child(1).observing("Calculate mean"): ts = anom.mean(dim=['lat', 'lon']) df = pd.DataFrame(data=ts[var].values, columns=[name], index=ts.time.values) retval = df.rolling(window=window, center=True).mean().dropna() if threshold is None: return retval retval['El Nino'] = pd.Series((retval[name] > threshold), index=retval.index) retval['La Nina'] = pd.Series((retval[name] < -threshold), index=retval.index) return retval
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None if region: region = PolygonLike.convert(region) if var_names: var_names = VarNamesLike.convert(var_names) paths = [] if time_range: time_series = list(self._files.values()) file_paths = list(self._files.keys()) for i in range(len(time_series)): if time_series[i]: if isinstance(time_series[i], Tuple) and \ time_series[i][0] >= time_range[0] and \ time_series[i][1] <= time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) elif isinstance(time_series[i], datetime) and \ time_range[0] <= time_series[i] < time_range[1]: paths.extend(self._resolve_file_path(file_paths[i])) else: for file in self._files.items(): paths.extend(self._resolve_file_path(file[0])) if paths: paths = sorted(set(paths)) try: ds = open_xarray_dataset(paths) if region: [lat_min, lon_min, lat_max, lon_max] = region.bounds ds = ds.sel(drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) if var_names: ds = ds.drop([ var_name for var_name in ds.variables.keys() if var_name not in var_names ]) return ds except OSError as e: raise IOError("Files: {} caused:\nOSError({}): {}".format( paths, e.errno, e.strerror)) else: return None
def test_convert(self): self.assertEqual(PolygonLike.convert(None), None) self.assertEqual(PolygonLike.convert(''), None) coords = [(10.4, 20.2), (30.8, 20.2), (30.8, 40.8), (10.4, 40.8)] self.assertTrue(PolygonLike.convert(coords), Polygon(coords)) self.assertTrue(PolygonLike.convert([10.4, 20.2, 30.8, 40.8]), Polygon(coords)) with self.assertRaises(ValidationError) as err: PolygonLike.convert('aaa') self.assertEqual(str(err.exception), "Value cannot be converted into a 'PolygonLike': " "Invalid geometry WKT format.")
def test_convert(self): self.assertEqual(PolygonLike.convert(None), None) self.assertEqual(PolygonLike.convert(''), None) coords = [(10.4, 20.2), (30.8, 20.2), (30.8, 40.8), (10.4, 40.8)] self.assertTrue(PolygonLike.convert(coords), Polygon(coords)) self.assertTrue(PolygonLike.convert([10.4, 20.2, 30.8, 40.8]), Polygon(coords)) with self.assertRaises(ValueError) as err: PolygonLike.convert('aaa') self.assertEqual(str(err.exception), 'cannot convert value <aaa> to PolygonLike')
def __init__(self, ds_id: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, meta_info: dict = None, status: DataSourceStatus = None): self._id = ds_id if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert(temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert(tuple([files_range[0][0], files_range[files_number - 1][1]])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert((files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert(spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert(variables) if variables else [] self._meta_info = meta_info if meta_info else OrderedDict() if self._variables and not self._meta_info.get('variables', None): self._meta_info['variables'] = [ {'name': var_name, 'units': '', 'long_name': '', 'standard_name': '' } for var_name in self._variables] self._status = status if status else DataSourceStatus.READY
def enso(ds: xr.Dataset, var: VarName.TYPE, file: str, region: str = 'n34', custom_region: PolygonLike.TYPE = None, threshold: float = None, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ Calculate ENSO index, which is defined as a five month running mean of anomalies of monthly means of SST data in the given region. :param ds: A monthly SST dataset :param file: Path to the reference data file e.g. a climatology. A suitable reference dataset can be generated using the long_term_average operation :param var: Dataset variable to use for index calculation :param region: Region for index calculation, the default is Nino3.4 :param custom_region: If 'custom' is chosen as the 'region', this parameter has to be provided to set the desired region. :param threshold: If given, boolean El Nino/La Nina timeseries will be calculated and added to the output dataset, according to the given threshold. Where anomaly larger than then positive value of the threshold indicates El Nino and anomaly smaller than the negative of the given threshold indicates La Nina. :param monitor: a progress monitor. :return: A dataset that contains the index timeseries. """ regions = { 'N1+2': '-90, -10, -80, 0', 'N3': '-150, -5, -90, 5', 'N3.4': '-170, -5, -120, 5', 'N4': '160, -5, -150, 5', 'custom': custom_region } converted_region = PolygonLike.convert(regions[region]) if not converted_region: raise ValidationError( 'No region has been provided to ENSO index calculation') name = 'ENSO ' + region + ' Index' if 'custom' == region: name = 'ENSO Index over ' + PolygonLike.format(converted_region) return _generic_index_calculation(ds, var, converted_region, 5, file, name, threshold, monitor)
def _generic_index_calculation(ds: xr.Dataset, var: VarName.TYPE, region: PolygonLike.TYPE, window: int, file: str, name: str, threshold: float = None, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ A generic index calculation. Where an index is defined as an anomaly against the given reference of a moving average of the given window size of the given given region of the given variable of the given dataset. :param ds: Dataset from which to calculate the index :param var: Variable from which to calculate index :param region: Spatial subset from which to calculate the index :param window: Window size for the moving average :param file: Path to the reference file :param threshold: Absolute threshold that indicates an ENSO event :param name: Name of the index :param monitor: a progress monitor. :return: A dataset that contains the index timeseries """ var = VarName.convert(var) region = PolygonLike.convert(region) with monitor.starting("Calculate the index", total_work=2): ds = select_var(ds, var) ds_subset = subset_spatial(ds, region) anom = anomaly_external(ds_subset, file, monitor=monitor.child(1)) with monitor.child(1).observing("Calculate mean"): ts = anom.mean(dim=['lat', 'lon']) df = pd.DataFrame(data=ts[var].values, columns=[name], index=ts.time) retval = df.rolling(window=window, center=True).mean().dropna() if threshold is None: return retval retval['El Nino'] = pd.Series((retval[name] > threshold), index=retval.index) retval['La Nina'] = pd.Series((retval[name] < -threshold), index=retval.index) return retval
def test_convert(self): self.assertEqual(PolygonLike.convert(None), None) self.assertEqual(PolygonLike.convert(''), None) coords = [(10.4, 20.2), (30.8, 20.2), (30.8, 40.8), (10.4, 40.8)] self.assertTrue(PolygonLike.convert(coords), Polygon(coords)) self.assertTrue(PolygonLike.convert([10.4, 20.2, 30.8, 40.8]), Polygon(coords)) with self.assertRaises(ValidationError) as err: PolygonLike.convert('aaa') self.assertEqual( str(err.exception), "Value cannot be converted into a 'PolygonLike': " "Invalid geometry WKT format.")
def __init__(self, name: str, files: Union[Sequence[str], OrderedDict], data_store: 'LocalDataStore', temporal_coverage: TimeRangeLike.TYPE = None, spatial_coverage: PolygonLike.TYPE = None, variables: VarNamesLike.TYPE = None, reference_type: str = None, reference_name: str = None): self._name = name if isinstance(files, Sequence): self._files = OrderedDict.fromkeys(files) else: self._files = files self._data_store = data_store initial_temporal_coverage = TimeRangeLike.convert( temporal_coverage) if temporal_coverage else None if not initial_temporal_coverage: files_number = len(self._files.items()) if files_number > 0: files_range = list(self._files.values()) if files_range: if isinstance(files_range[0], Tuple): initial_temporal_coverage = TimeRangeLike.convert( tuple([ files_range[0][0], files_range[files_number - 1][1] ])) elif isinstance(files_range[0], datetime): initial_temporal_coverage = TimeRangeLike.convert( (files_range[0], files_range[files_number - 1])) self._temporal_coverage = initial_temporal_coverage self._spatial_coverage = PolygonLike.convert( spatial_coverage) if spatial_coverage else None self._variables = VarNamesLike.convert( variables) if variables else None self._reference_type = reference_type if reference_type else None self._reference_name = reference_name
def enso(ds: xr.Dataset, var: VarName.TYPE, file: str, region: str = 'n34', custom_region: PolygonLike.TYPE = None, threshold: float = None, monitor: Monitor = Monitor.NONE) -> pd.DataFrame: """ Calculate ENSO index, which is defined as a five month running mean of anomalies of monthly means of SST data in the given region. :param ds: A monthly SST dataset :param file: Path to the reference data file e.g. a climatology. A suitable reference dataset can be generated using the long_term_average operation :param var: Dataset variable to use for index calculation :param region: Region for index calculation, the default is Nino3.4 :param custom_region: If 'custom' is chosen as the 'region', this parameter has to be provided to set the desired region. :param threshold: If given, boolean El Nino/La Nina timeseries will be calculated and added to the output dataset, according to the given threshold. Where anomaly larger than then positive value of the threshold indicates El Nino and anomaly smaller than the negative of the given threshold indicates La Nina. :param monitor: a progress monitor. :return: A dataset that contains the index timeseries. """ regions = {'N1+2': '-90, -10, -80, 0', 'N3': '-150, -5, -90, 5', 'N3.4': '-170, -5, -120, 5', 'N4': '160, -5, -150, 5', 'custom': custom_region} converted_region = PolygonLike.convert(regions[region]) if not converted_region: raise ValidationError('No region has been provided to ENSO index calculation') name = 'ENSO ' + region + ' Index' if 'custom' == region: name = 'ENSO Index over ' + PolygonLike.format(converted_region) return _generic_index_calculation(ds, var, converted_region, 5, file, name, threshold, monitor)
def open_dataset(self, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, protocol: str = None) -> Any: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'Data source \'{}\' does not seem to have any data files'.format( self.name) if time_range is not None: msg += ' in given time range {}'.format( TimeRangeLike.format(time_range)) raise IOError(msg) files = self._get_urls_list(selected_file_list, _ODP_PROTOCOL_OPENDAP) try: ds = open_xarray_dataset(files) if region: [lat_min, lon_min, lat_max, lon_max] = region.bounds ds = ds.sel(drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) if var_names: ds = ds.drop([ var_name for var_name in ds.variables.keys() if var_name not in var_names ]) return ds except OSError as e: raise IOError("Files: {} caused:\nOSError({}): {}".format( files, e.errno, e.strerror))
def test_make_local_and_update(self): soilmoisture_data_sources = self.data_store.query( query_expr='esacci.SOILMOISTURE.day.L3S.SSMV.multi-sensor.multi-platform.COMBINED.02-1.r1') soilmoisture_data_source = soilmoisture_data_sources[0] reference_path = os.path.join(os.path.dirname(__file__), os.path.normpath('resources/datasources/local/files/')) def find_files_mock(_, time_range): def build_file_item(item_name: str, date_from: datetime, date_to: datetime, size: int): return [item_name, date_from, date_to, size, {'OPENDAP': os.path.join(reference_path, item_name), 'HTTPServer': 'file:' + urllib.request.pathname2url(os.path.join(reference_path, item_name))}] reference_files = { 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781114000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 14, 0, 0), 'date_to': datetime.datetime(1978, 11, 14, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781115000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 15, 0, 0), 'date_to': datetime.datetime(1978, 11, 15, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781116000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 16, 0, 0), 'date_to': datetime.datetime(1978, 11, 16, 23, 59), 'size': 21511378 } } reference_files_list = [] for reference_file in reference_files.items(): file_name = reference_file[0] file_date_from = reference_file[1].get('date_from') file_date_to = reference_file[1].get('date_to') file_size = reference_file[1].get('size') if time_range: if file_date_from >= time_range[0] and file_date_to <= time_range[1]: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) else: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) return reference_files_list with unittest.mock.patch('cate.ds.esa_cci_odp.EsaCciOdpDataSource._find_files', find_files_mock): with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'local_ds_test' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) try: new_ds = soilmoisture_data_source.make_local(new_ds_title, time_range=new_ds_time_range) except Exception: raise ValueError(reference_path, os.listdir(reference_path)) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), new_ds_time_range) new_ds_w_one_variable_title = 'local_ds_test_var' new_ds_w_one_variable_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_one_variable_var_names = VarNamesLike.convert(['sm']) new_ds_w_one_variable = soilmoisture_data_source.make_local( new_ds_w_one_variable_title, time_range=new_ds_w_one_variable_time_range, var_names=new_ds_w_one_variable_var_names ) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_w_one_variable_title) ds = new_ds_w_one_variable.open_dataset() new_ds_w_one_variable_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(ds.variables), set(new_ds_w_one_variable_var_names)) new_ds_w_region_title = 'from_local_to_local_region' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) new_ds_w_region_title = 'from_local_to_local_region_one_var' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) new_ds_w_region_title = 'from_local_to_local_region_two_var_sm_uncertainty' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm', 'sm_uncertainty']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) # type: LocalDataSource self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) empty_ds_timerange = (datetime.datetime(2017, 12, 1, 0, 0), datetime.datetime(2017, 12, 31, 23, 59)) with self.assertRaises(DataAccessError) as cm: soilmoisture_data_source.make_local('empty_ds', time_range=empty_ds_timerange) self.assertEqual(f'Data source "{soilmoisture_data_source.id}" does not' f' seem to have any datasets in given' f' time range {TimeRangeLike.format(empty_ds_timerange)}', str(cm.exception)) new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 14, 23, 59))) new_ds = soilmoisture_data_source.make_local("title_test_copy", time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], soilmoisture_data_source.meta_info['title']) title = "Title Test!" new_ds = soilmoisture_data_source.make_local("title_test_set", title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], title)
def make_local(self, local_name: str, local_id: str = None, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE) -> Optional[DataSource]: time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert(var_names) if var_names else None ds_id = local_name title = local_id local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: add_to_data_store_registry() local_store = DATA_STORE_REGISTRY.get_data_store('local') if not local_store: raise ValueError('Cannot initialize `local` DataStore') uuid = LocalDataStore.generate_uuid(ref_id=self.id, time_range=time_range, region=region, var_names=var_names) if not ds_id or len(ds_id) == 0: ds_id = "local.{}.{}".format(self.id, uuid) existing_ds_list = local_store.query(ds_id=ds_id) if len(existing_ds_list) == 1: return existing_ds_list[0] else: existing_ds_list = local_store.query(ds_id='local.%s' % ds_id) if len(existing_ds_list) == 1: if existing_ds_list[0].meta_info.get('uuid', None) == uuid: return existing_ds_list[0] else: raise ValueError( 'Datastore {} already contains dataset {}'.format( local_store.id, ds_id)) local_meta_info = self.meta_info.copy() local_meta_info['ref_uuid'] = local_meta_info.get('uuid', None) local_meta_info['uuid'] = uuid local_ds = local_store.create_data_source(ds_id, title=title, time_range=time_range, region=region, var_names=var_names, meta_info=local_meta_info, lock_file=True) if local_ds: if not local_ds.is_complete: try: self._make_local(local_ds, time_range, region, var_names, monitor=monitor) except Cancellation as c: local_store.remove_data_source(local_ds) raise c except Exception as e: if local_ds.is_empty: local_store.remove_data_source(local_ds) raise e if local_ds.is_empty: local_store.remove_data_source(local_ds) return None local_store.register_ds(local_ds) return local_ds else: return None
def _make_local(self, local_ds: LocalDataSource, time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): local_id = local_ds.id time_range = TimeRangeLike.convert(time_range) region = PolygonLike.convert(region) var_names = VarNamesLike.convert(var_names) time_range, region, var_names = self._apply_make_local_fixes( time_range, region, var_names) compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False do_update_of_verified_time_coverage_start_once = True verified_time_coverage_start = None verified_time_coverage_end = None encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) if region or var_names: protocol = _ODP_PROTOCOL_OPENDAP else: protocol = _ODP_PROTOCOL_HTTP local_path = os.path.join(local_ds.data_store.data_store_path, local_id) if not os.path.exists(local_path): os.makedirs(local_path) selected_file_list = self._find_files(time_range) if not selected_file_list: msg = 'CCI Open Data Portal data source "{}"\ndoes not seem to have any datasets'.format( self.id) if time_range is not None: msg += ' in given time range {}'.format( TimeRangeLike.format(time_range)) raise DataAccessError(msg) try: if protocol == _ODP_PROTOCOL_OPENDAP: do_update_of_variables_meta_info_once = True do_update_of_region_meta_info_once = True files = self._get_urls_list(selected_file_list, protocol) monitor.start('Sync ' + self.id, total_work=len(files)) for idx, dataset_uri in enumerate(files): child_monitor = monitor.child(work=1) file_name = os.path.basename(dataset_uri) local_filepath = os.path.join(local_path, file_name) time_coverage_start = selected_file_list[idx][1] time_coverage_end = selected_file_list[idx][2] try: child_monitor.start(label=file_name, total_work=1) remote_dataset = xr.open_dataset(dataset_uri) if var_names: remote_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.data_vars.keys() if var_name not in var_names ]) if region: remote_dataset = normalize_impl(remote_dataset) remote_dataset = subset_spatial_impl( remote_dataset, region) geo_lon_min, geo_lat_min, geo_lon_max, geo_lat_max = region.bounds remote_dataset.attrs[ 'geospatial_lat_min'] = geo_lat_min remote_dataset.attrs[ 'geospatial_lat_max'] = geo_lat_max remote_dataset.attrs[ 'geospatial_lon_min'] = geo_lon_min remote_dataset.attrs[ 'geospatial_lon_max'] = geo_lon_max if do_update_of_region_meta_info_once: local_ds.meta_info['bbox_maxx'] = geo_lon_max local_ds.meta_info['bbox_minx'] = geo_lon_min local_ds.meta_info['bbox_maxy'] = geo_lat_max local_ds.meta_info['bbox_miny'] = geo_lat_min do_update_of_region_meta_info_once = False if compression_enabled: for sel_var_name in remote_dataset.variables.keys( ): remote_dataset.variables.get( sel_var_name).encoding.update( encoding_update) remote_dataset.to_netcdf(local_filepath) child_monitor.progress(work=1, msg=str(time_coverage_start)) finally: if do_update_of_variables_meta_info_once: variables_info = local_ds.meta_info.get( 'variables', []) local_ds.meta_info['variables'] = [ var_info for var_info in variables_info if var_info.get('name') in remote_dataset. variables.keys() and var_info.get( 'name') not in remote_dataset.dims.keys() ] do_update_of_variables_meta_info_once = False local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) if do_update_of_verified_time_coverage_start_once: verified_time_coverage_start = time_coverage_start do_update_of_verified_time_coverage_start_once = False verified_time_coverage_end = time_coverage_end child_monitor.done() else: outdated_file_list = [] for file_rec in selected_file_list: filename, _, _, file_size, url = file_rec dataset_file = os.path.join(local_path, filename) # todo (forman, 20160915): must perform better checks on dataset_file if it is... # ... outdated or incomplete or corrupted. # JSON also includes "checksum" and "checksum_type" fields. if not os.path.isfile(dataset_file) or ( file_size and os.path.getsize(dataset_file) != file_size): outdated_file_list.append(file_rec) if outdated_file_list: with monitor.starting('Sync ' + self.id, len(outdated_file_list)): bytes_to_download = sum( [file_rec[3] for file_rec in outdated_file_list]) dl_stat = _DownloadStatistics(bytes_to_download) file_number = 1 for filename, coverage_from, coverage_to, file_size, url in outdated_file_list: dataset_file = os.path.join(local_path, filename) sub_monitor = monitor.child(work=1.0) # noinspection PyUnusedLocal def reporthook(block_number, read_size, total_file_size): dl_stat.handle_chunk(read_size) sub_monitor.progress(work=read_size, msg=str(dl_stat)) sub_monitor_msg = "file %d of %d" % ( file_number, len(outdated_file_list)) with sub_monitor.starting(sub_monitor_msg, file_size): urllib.request.urlretrieve( url[protocol], filename=dataset_file, reporthook=reporthook) file_number += 1 local_ds.add_dataset( os.path.join(local_id, filename), (coverage_from, coverage_to)) if do_update_of_verified_time_coverage_start_once: verified_time_coverage_start = coverage_from do_update_of_verified_time_coverage_start_once = False verified_time_coverage_end = coverage_to except OSError as e: raise DataAccessError( "Copying remote data source failed: {}".format(e), source=self) from e local_ds.meta_info['temporal_coverage_start'] = TimeLike.format( verified_time_coverage_start) local_ds.meta_info['temporal_coverage_end'] = TimeLike.format( verified_time_coverage_end) local_ds.save(True)
def test_format(self): coords = [(0.0, 0.0), (0.0, 1.0), (1.0, 1.0), (1.0, 0.0)] pol = PolygonLike.convert(coords) self.assertTrue( 'POLYGON ((0 0, 0 1, 1 1, 1 0, 0 0))' == PolygonLike.format(pol))
def test_make_local_and_update(self): reference_path = os.path.join(os.path.dirname(__file__), os.path.normpath('resources/datasources/local/files/')) def find_files_mock(_, time_range): def build_file_item(item_name: str, date_from: datetime, date_to: datetime, size: int): return [item_name, date_from, date_to, size, {'OPENDAP': os.path.join(reference_path, item_name), 'HTTPServer': 'file:' + urllib.request.pathname2url(os.path.join(reference_path, item_name))}] reference_files = { 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781114000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 14, 0, 0), 'date_to': datetime.datetime(1978, 11, 14, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781115000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 15, 0, 0), 'date_to': datetime.datetime(1978, 11, 15, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781116000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 16, 0, 0), 'date_to': datetime.datetime(1978, 11, 16, 23, 59), 'size': 21511378 } } reference_files_list = [] for reference_file in reference_files.items(): file_name = reference_file[0] file_date_from = reference_file[1].get('date_from') file_date_to = reference_file[1].get('date_to') file_size = reference_file[1].get('size') if time_range: if file_date_from >= time_range[0] and file_date_to <= time_range[1]: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) else: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) return reference_files_list with unittest.mock.patch('cate.ds.esa_cci_odp.EsaCciOdpDataSource._find_files', find_files_mock): with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): try: new_ds = self.data_source.make_local('local_ds_test', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) except: raise ValueError(reference_path, os.listdir(reference_path)) self.assertEqual(new_ds.name, 'local.local_ds_test') self.assertEqual(new_ds.temporal_coverage(), (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59))) self.data_source.update_local(new_ds.name, (datetime.datetime(1978, 11, 15, 00, 00), datetime.datetime(1978, 11, 16, 23, 59))) self.assertEqual(new_ds.temporal_coverage(), TimeRangeLike.convert( (datetime.datetime(1978, 11, 15, 0, 0), datetime.datetime(1978, 11, 16, 23, 59)))) self.data_source.update_local(new_ds.name, (datetime.datetime(1978, 11, 14, 00, 00), datetime.datetime(1978, 11, 15, 23, 59))) self.assertEqual(new_ds.temporal_coverage(), TimeRangeLike.convert( (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)))) with self.assertRaises(ValueError) as context: self.data_source.update_local("wrong_ds_name", (datetime.datetime(1978, 11, 15, 00, 00), datetime.datetime(1978, 11, 16, 23, 59))) self.assertTrue("Couldn't find local DataSource", context.exception.args[0]) new_ds_w_one_variable = self.data_source.make_local( 'local_ds_test_2', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)), None, ['sm']) self.assertEqual(new_ds_w_one_variable.name, 'local.local_ds_test_2') ds = new_ds_w_one_variable.open_dataset() self.assertSetEqual(set(ds.variables), {'sm', 'lat', 'lon', 'time'}) new_ds_w_region = self.data_source.make_local( 'from_local_to_local_region', None, (datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 15, 23, 59)), "10,10,20,20", ['sm']) # type: LocalDataSource self.assertEqual(new_ds_w_region.name, 'local.from_local_to_local_region') self.assertEqual(new_ds_w_region.spatial_coverage(), PolygonLike.convert("10,10,20,20")) data_set = new_ds_w_region.open_dataset() self.assertSetEqual(set(data_set.variables), {'sm', 'lat', 'lon', 'time'})
def plot_map(ds: xr.Dataset, var: VarName.TYPE = None, indexers: DictLike.TYPE = None, time: TimeLike.TYPE = None, region: PolygonLike.TYPE = None, projection: str = 'PlateCarree', central_lon: float = 0.0, title: str = None, properties: DictLike.TYPE = None, file: str = None) -> Figure: """ Create a geographic map plot for the variable given by dataset *ds* and variable name *var*. Plots the given variable from the given dataset on a map with coastal lines. In case no variable name is given, the first encountered variable in the dataset is plotted. In case no *time* is given, the first time slice is taken. It is also possible to set extents of the plot. If no extents are given, a global plot is created. The plot can either be shown using pyplot functionality, or saved, if a path is given. The following file formats for saving the plot are supported: eps, jpeg, jpg, pdf, pgf, png, ps, raw, rgba, svg, svgz, tif, tiff :param ds: the dataset containing the variable to plot :param var: the variable's name :param indexers: Optional indexers into data array of *var*. The *indexers* is a dictionary or a comma-separated string of key-value pairs that maps the variable's dimension names to constant labels. e.g. "layer=4". :param time: time slice index to plot, can be a string "YYYY-MM-DD" or an integer number :param region: Region to plot :param projection: name of a global projection, see http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html :param central_lon: central longitude of the projection in degrees :param title: an optional title :param properties: optional plot properties for Python matplotlib, e.g. "bins=512, range=(-1.5, +1.5)" For full reference refer to https://matplotlib.org/api/lines_api.html and https://matplotlib.org/api/_as_gen/matplotlib.axes.Axes.contourf.html :param file: path to a file in which to save the plot :return: a matplotlib figure object or None if in IPython mode """ if not isinstance(ds, xr.Dataset): raise NotImplementedError( 'Only gridded datasets are currently supported') var_name = None if not var: for key in ds.data_vars.keys(): var_name = key break else: var_name = VarName.convert(var) var = ds[var_name] time = TimeLike.convert(time) indexers = DictLike.convert(indexers) or {} properties = DictLike.convert(properties) or {} extents = None region = PolygonLike.convert(region) if region: lon_min, lat_min, lon_max, lat_max = region.bounds if not _check_bounding_box(lat_min, lat_max, lon_min, lon_max): raise ValueError( 'Provided plot extents do not form a valid bounding box ' 'within [-180.0,+180.0,-90.0,+90.0]') extents = [lon_min, lon_max, lat_min, lat_max] # See http://scitools.org.uk/cartopy/docs/v0.15/crs/projections.html# if projection == 'PlateCarree': proj = ccrs.PlateCarree(central_longitude=central_lon) elif projection == 'LambertCylindrical': proj = ccrs.LambertCylindrical(central_longitude=central_lon) elif projection == 'Mercator': proj = ccrs.Mercator(central_longitude=central_lon) elif projection == 'Miller': proj = ccrs.Miller(central_longitude=central_lon) elif projection == 'Mollweide': proj = ccrs.Mollweide(central_longitude=central_lon) elif projection == 'Orthographic': proj = ccrs.Orthographic(central_longitude=central_lon) elif projection == 'Robinson': proj = ccrs.Robinson(central_longitude=central_lon) elif projection == 'Sinusoidal': proj = ccrs.Sinusoidal(central_longitude=central_lon) elif projection == 'NorthPolarStereo': proj = ccrs.NorthPolarStereo(central_longitude=central_lon) elif projection == 'SouthPolarStereo': proj = ccrs.SouthPolarStereo(central_longitude=central_lon) else: raise ValueError('illegal projection: "%s"' % projection) figure = plt.figure(figsize=(8, 4)) ax = plt.axes(projection=proj) if extents: ax.set_extent(extents) else: ax.set_global() ax.coastlines() var_data = _get_var_data(var, indexers, time=time, remaining_dims=('lon', 'lat')) var_data.plot.contourf(ax=ax, transform=proj, **properties) if title: ax.set_title(title) figure.tight_layout() if file: figure.savefig(file) return figure if not in_notebook() else None
def test_make_local_and_update(self): soilmoisture_data_sources = self.data_store.query( query_expr='esacci.SOILMOISTURE.day.L3S.SSMV.multi-sensor.multi-platform.COMBINED.02-1.r1') soilmoisture_data_source = soilmoisture_data_sources[0] reference_path = os.path.join(os.path.dirname(__file__), os.path.normpath('resources/datasources/local/files/')) def find_files_mock(_, time_range): def build_file_item(item_name: str, date_from: datetime, date_to: datetime, size: int): return [item_name, date_from, date_to, size, {'OPENDAP': os.path.join(reference_path, item_name), 'HTTPServer': 'file:' + urllib.request.pathname2url(os.path.join(reference_path, item_name))}] reference_files = { 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781114000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 14, 0, 0), 'date_to': datetime.datetime(1978, 11, 14, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781115000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 15, 0, 0), 'date_to': datetime.datetime(1978, 11, 15, 23, 59), 'size': 21511378 }, 'ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-19781116000000-fv02.2.nc': { 'date_from': datetime.datetime(1978, 11, 16, 0, 0), 'date_to': datetime.datetime(1978, 11, 16, 23, 59), 'size': 21511378 } } reference_files_list = [] for reference_file in reference_files.items(): file_name = reference_file[0] file_date_from = reference_file[1].get('date_from') file_date_to = reference_file[1].get('date_to') file_size = reference_file[1].get('size') if time_range: if file_date_from >= time_range[0] and file_date_to <= time_range[1]: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) else: reference_files_list.append(build_file_item(file_name, file_date_from, file_date_to, file_size)) return reference_files_list with unittest.mock.patch('cate.ds.esa_cci_odp.EsaCciOdpDataSource._find_files', find_files_mock): with unittest.mock.patch.object(EsaCciOdpDataStore, 'query', return_value=[]): new_ds_title = 'local_ds_test' new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) try: new_ds = soilmoisture_data_source.make_local(new_ds_title, time_range=new_ds_time_range) except Exception: raise ValueError(reference_path, os.listdir(reference_path)) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.id, "local.%s" % new_ds_title) self.assertEqual(new_ds.temporal_coverage(), new_ds_time_range) new_ds_w_one_variable_title = 'local_ds_test_var' new_ds_w_one_variable_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_one_variable_var_names = VarNamesLike.convert(['sm']) new_ds_w_one_variable = soilmoisture_data_source.make_local( new_ds_w_one_variable_title, time_range=new_ds_w_one_variable_time_range, var_names=new_ds_w_one_variable_var_names ) self.assertIsNotNone(new_ds_w_one_variable) self.assertEqual(new_ds_w_one_variable.id, "local.%s" % new_ds_w_one_variable_title) ds = new_ds_w_one_variable.open_dataset() new_ds_w_one_variable_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(ds.variables), set(new_ds_w_one_variable_var_names)) new_ds_w_region_title = 'from_local_to_local_region' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) new_ds_w_region_title = 'from_local_to_local_region_one_var' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) new_ds_w_region_title = 'from_local_to_local_region_two_var_sm_uncertainty' new_ds_w_region_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 16, 23, 59))) new_ds_w_region_var_names = VarNamesLike.convert(['sm', 'sm_uncertainty']) new_ds_w_region_spatial_coverage = PolygonLike.convert("10,20,30,40") new_ds_w_region = soilmoisture_data_source.make_local( new_ds_w_region_title, time_range=new_ds_w_region_time_range, var_names=new_ds_w_region_var_names, region=new_ds_w_region_spatial_coverage) self.assertIsNotNone(new_ds_w_region) self.assertEqual(new_ds_w_region.id, "local.%s" % new_ds_w_region_title) self.assertEqual(new_ds_w_region.spatial_coverage(), new_ds_w_region_spatial_coverage) data_set = new_ds_w_region.open_dataset() new_ds_w_region_var_names.extend(['lat', 'lon', 'time']) self.assertSetEqual(set(data_set.variables), set(new_ds_w_region_var_names)) empty_ds_timerange = (datetime.datetime(2017, 12, 1, 0, 0), datetime.datetime(2017, 12, 31, 23, 59)) with self.assertRaises(DataAccessError) as cm: soilmoisture_data_source.make_local('empty_ds', time_range=empty_ds_timerange) self.assertEqual(f'Data source "{soilmoisture_data_source.id}" does not' f' seem to have any datasets in given' f' time range {TimeRangeLike.format(empty_ds_timerange)}', str(cm.exception)) new_ds_time_range = TimeRangeLike.convert((datetime.datetime(1978, 11, 14, 0, 0), datetime.datetime(1978, 11, 14, 23, 59))) new_ds = soilmoisture_data_source.make_local("title_test_copy", time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], soilmoisture_data_source.meta_info['title']) title = "Title Test!" new_ds = soilmoisture_data_source.make_local("title_test_set", title, time_range=new_ds_time_range) self.assertIsNotNone(new_ds) self.assertEqual(new_ds.meta_info['title'], title)
def _make_local(self, local_ds: 'LocalDataSource', time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): # local_name = local_ds.name local_id = local_ds.name time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) local_path = os.path.join(local_ds.data_store.data_store_path, local_id) data_store_path = local_ds.data_store.data_store_path if not os.path.exists(local_path): os.makedirs(local_path) monitor.start("Sync " + self.name, total_work=len(self._files.items())) for remote_relative_filepath, coverage in self._files.items(): child_monitor = monitor.child(work=1) file_name = os.path.basename(remote_relative_filepath) local_relative_filepath = os.path.join(local_id, file_name) local_absolute_filepath = os.path.join(data_store_path, local_relative_filepath) remote_absolute_filepath = os.path.join( self._data_store.data_store_path, remote_relative_filepath) if isinstance(coverage, Tuple): time_coverage_start = coverage[0] time_coverage_end = coverage[1] remote_netcdf = None local_netcdf = None if not time_range or time_coverage_start >= time_range[ 0] and time_coverage_end <= time_range[1]: if region or var_names: try: remote_netcdf = NetCDF4DataStore( remote_absolute_filepath) local_netcdf = NetCDF4DataStore( local_absolute_filepath, mode='w', persist=True) local_netcdf.set_attributes( remote_netcdf.get_attrs()) remote_dataset = xr.Dataset.load_store( remote_netcdf) process_region = False if region: geo_lat_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_min') geo_lat_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_max') geo_lon_min = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_min') geo_lon_max = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_max') geo_lat_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lon_resolution') geo_lon_res = self._get_harmonized_coordinate_value( remote_dataset.attrs, 'geospatial_lat_resolution') if not (isnan(geo_lat_min) or isnan(geo_lat_max) or isnan(geo_lon_min) or isnan(geo_lon_max) or isnan(geo_lat_res) or isnan(geo_lon_res)): process_region = True [lat_min, lon_min, lat_max, lon_max] = region.bounds lat_min = floor( (lat_min - geo_lat_min) / geo_lat_res) lat_max = ceil( (lat_max - geo_lat_min) / geo_lat_res) lon_min = floor( (lon_min - geo_lon_min) / geo_lon_res) lon_max = ceil( (lon_max - geo_lon_min) / geo_lon_res) # TODO (kbernat): check why dataset.sel fails! remote_dataset = remote_dataset.isel( drop=False, lat=slice(lat_min, lat_max), lon=slice(lon_min, lon_max)) geo_lat_max = lat_max * geo_lat_res + geo_lat_min geo_lat_min += lat_min * geo_lat_res geo_lon_max = lon_max * geo_lon_res + geo_lon_min geo_lon_min += lon_min * geo_lon_res if not var_names: var_names = [ var_name for var_name in remote_netcdf.variables.keys() ] var_names.extend([ coord_name for coord_name in remote_dataset.coords.keys() if coord_name not in var_names ]) child_monitor.start(label=file_name, total_work=len(var_names)) for sel_var_name in var_names: var_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.variables.keys() if var_name != sel_var_name ]) if compression_enabled: var_dataset.variables.get( sel_var_name).encoding.update( encoding_update) local_netcdf.store_dataset(var_dataset) child_monitor.progress(work=1, msg=sel_var_name) if process_region: local_netcdf.set_attribute( 'geospatial_lat_min', geo_lat_min) local_netcdf.set_attribute( 'geospatial_lat_max', geo_lat_max) local_netcdf.set_attribute( 'geospatial_lon_min', geo_lon_min) local_netcdf.set_attribute( 'geospatial_lon_max', geo_lon_max) finally: if remote_netcdf: remote_netcdf.close() if local_netcdf: local_netcdf.close() local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() else: shutil.copy(remote_absolute_filepath, local_absolute_filepath) local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() monitor.done() return local_id
def _make_local(self, local_ds: 'LocalDataSource', time_range: TimeRangeLike.TYPE = None, region: PolygonLike.TYPE = None, var_names: VarNamesLike.TYPE = None, monitor: Monitor = Monitor.NONE): local_id = local_ds.id time_range = TimeRangeLike.convert(time_range) if time_range else None region = PolygonLike.convert(region) if region else None var_names = VarNamesLike.convert( var_names) if var_names else None # type: Sequence compression_level = get_config_value('NETCDF_COMPRESSION_LEVEL', NETCDF_COMPRESSION_LEVEL) compression_enabled = True if compression_level > 0 else False encoding_update = dict() if compression_enabled: encoding_update.update({ 'zlib': True, 'complevel': compression_level }) local_path = os.path.join(local_ds.data_store.data_store_path, local_id) data_store_path = local_ds.data_store.data_store_path if not os.path.exists(local_path): os.makedirs(local_path) monitor.start("Sync " + self.id, total_work=len(self._files.items())) for remote_relative_filepath, coverage in self._files.items(): child_monitor = monitor.child(work=1) file_name = os.path.basename(remote_relative_filepath) local_relative_filepath = os.path.join(local_id, file_name) local_absolute_filepath = os.path.join(data_store_path, local_relative_filepath) remote_absolute_filepath = os.path.join( self._data_store.data_store_path, remote_relative_filepath) if isinstance(coverage, Tuple): time_coverage_start = coverage[0] time_coverage_end = coverage[1] if not time_range or time_coverage_start >= time_range[ 0] and time_coverage_end <= time_range[1]: if region or var_names: do_update_of_variables_meta_info_once = True do_update_of_region_meta_info_once = True try: remote_dataset = xr.open_dataset( remote_absolute_filepath) if var_names: remote_dataset = remote_dataset.drop([ var_name for var_name in remote_dataset.data_vars.keys() if var_name not in var_names ]) if region: remote_dataset = normalize_impl(remote_dataset) remote_dataset = subset_spatial_impl( remote_dataset, region) geo_lon_min, geo_lat_min, geo_lon_max, geo_lat_max = region.bounds remote_dataset.attrs[ 'geospatial_lat_min'] = geo_lat_min remote_dataset.attrs[ 'geospatial_lat_max'] = geo_lat_max remote_dataset.attrs[ 'geospatial_lon_min'] = geo_lon_min remote_dataset.attrs[ 'geospatial_lon_max'] = geo_lon_max if do_update_of_region_meta_info_once: local_ds.meta_info[ 'bbox_maxx'] = geo_lon_max local_ds.meta_info[ 'bbox_minx'] = geo_lon_min local_ds.meta_info[ 'bbox_maxy'] = geo_lat_max local_ds.meta_info[ 'bbox_miny'] = geo_lat_min do_update_of_region_meta_info_once = False if compression_enabled: for sel_var_name in remote_dataset.variables.keys( ): remote_dataset.variables.get( sel_var_name).encoding.update( encoding_update) remote_dataset.to_netcdf(local_absolute_filepath) child_monitor.progress( work=1, msg=str(time_coverage_start)) finally: if do_update_of_variables_meta_info_once: variables_info = local_ds.meta_info.get( 'variables', []) local_ds.meta_info['variables'] = [ var_info for var_info in variables_info if var_info.get('name') in remote_dataset. variables.keys() and var_info.get('name') not in remote_dataset.dims.keys() ] do_update_of_variables_meta_info_once = False local_ds.add_dataset( os.path.join(local_id, file_name), (time_coverage_start, time_coverage_end)) child_monitor.done() else: shutil.copy(remote_absolute_filepath, local_absolute_filepath) local_ds.add_dataset( local_relative_filepath, (time_coverage_start, time_coverage_end)) child_monitor.done() monitor.done() return local_id
def test_format(self): self.assertEqual(PolygonLike.format(None), '') coords = [(10.4, 20.2), (30.8, 20.2), (30.8, 40.8), (10.4, 40.8)] pol = PolygonLike.convert(coords) self.assertEqual(PolygonLike.format(pol), 'POLYGON ((10.4 20.2, 30.8 20.2, 30.8 40.8, 10.4 40.8, 10.4 20.2))')