def test_decode_standard_calendar_multidim_time_inside_timestamp_range( calendar, enable_cftimeindex): if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') noleap_time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar) noleap_time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar) mdim_time = np.empty((len(noleap_time1), 2), ) mdim_time[:, 0] = noleap_time1 mdim_time[:, 1] = noleap_time2 expected1 = times1.values expected2 = times2.values actual = coding.times.decode_cf_datetime( mdim_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) assert actual.dtype == np.dtype('M8[ns]') abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, 's')).all() assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def test_decode_standard_calendar_multidim_time_inside_timestamp_range( calendar): import cftime units = "days since 0001-01-01" times1 = pd.date_range("2001-04-01", end="2001-04-05", freq="D") times2 = pd.date_range("2001-05-01", end="2001-05-05", freq="D") time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar) time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar) mdim_time = np.empty((len(time1), 2)) mdim_time[:, 0] = time1 mdim_time[:, 1] = time2 expected1 = times1.values expected2 = times2.values actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype("M8[ns]") abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, "s")).all() assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def test_decode_multidim_time_outside_timestamp_range(calendar): from datetime import datetime import cftime units = "days since 0001-01-01" times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] time1 = cftime.date2num(times1, units, calendar=calendar) time2 = cftime.date2num(times2, units, calendar=calendar) mdim_time = np.empty((len(time1), 2)) mdim_time[:, 0] = time1 mdim_time[:, 1] = time2 expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True) expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True) with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") actual = coding.times.decode_cf_datetime(mdim_time, units, calendar=calendar) assert actual.dtype == np.dtype("O") abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, "s")).all() assert (abs_diff2 <= np.timedelta64(1, "s")).all()
def time_bnds_mon(years): return xr.DataArray(np.array([[ cftime.date2num(cftime.DatetimeNoLeap(year, mon, 1), units='days since 0001-01-01', calendar='noleap') - 1, cftime.date2num(cftime.DatetimeNoLeap(year, mon, eom_day_noleap[mon - 1]), units='days since 0001-01-01', calendar='noleap') ] for year, mon in product(years, range(1, 13))]), dims=('time', 'ntb'))
def restore_dataset(self, ds, attrs={}, encoding={}): """Return the original time variable to decoded or undecoded state. """ ds = xr.merge( ( ds, self._ds_time_computed.drop( [v for v in self._ds.variables if v not in self.static_variables] ), ) ) # get the time data from dataset time_data = ds[self.time_coord_name].data # if time was not originally decoded, return the dataset with time # un-decoded if not self.time_orig_decoded and self.isdecoded(time_data): time_data = cftime.date2num( time_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar'] ) elif self.time_orig_decoded and not self.isdecoded(time_data): time_data = cftime.num2date( time_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar'] ) ds[self.time_coord_name].attrs = self.time_attrs ds[self.time_coord_name].data = time_data.astype(self.time.dtype) if self.time_bound is not None: time_bound_data = ds[self.tb_name].data if not self.time_orig_decoded and self.isdecoded(time_bound_data): time_bound_data = cftime.date2num( time_bound_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar'], ) elif self.time_orig_decoded and not self.isdecoded(time_bound_data): time_bound_data = cftime.num2date( time_bound_data, units=self.time_attrs['units'], calendar=self.time_attrs['calendar'], ) ds[self.tb_name].data = time_bound_data ds[self.tb_name].attrs = self.time_bound_attrs return self.update_metadata(ds, new_attrs=attrs, new_encoding=encoding)
def test_decode_multidim_time_outside_timestamp_range(calendar, enable_cftimeindex): from datetime import datetime if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] noleap_time1 = cftime.date2num(times1, units, calendar=calendar) noleap_time2 = cftime.date2num(times2, units, calendar=calendar) mdim_time = np.empty((len(noleap_time1), 2), ) mdim_time[:, 0] = noleap_time1 mdim_time[:, 1] = noleap_time2 if enable_cftimeindex: expected1 = cftime.num2date(noleap_time1, units, calendar, only_use_cftime_datetimes=True) expected2 = cftime.num2date(noleap_time2, units, calendar, only_use_cftime_datetimes=True) else: expected1 = cftime.num2date(noleap_time1, units, calendar) expected2 = cftime.num2date(noleap_time2, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( mdim_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) assert actual.dtype == np.dtype('O') abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, 's')).all() assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def time_set_mid(ds, time_name, deep=False): """ Return copy of ds with values of ds[time_name] replaced with midpoints of ds[time_name].attrs['bounds'], if bounds attribute exists. Except for time_name, the returned Dataset is a copy of ds2. The copy is deep or not depending on the argument deep. """ ds_out = ds.copy(deep) if "bounds" not in ds[time_name].attrs: return ds_out tb_name = ds[time_name].attrs["bounds"] tb = ds[tb_name] bounds_dim = next(dim for dim in tb.dims if dim != time_name) # Use da = da.copy(data=...), in order to preserve attributes and encoding. # If tb is an array of datetime objects then encode time before averaging. # Do this because computing the mean on datetime objects with xarray fails # if the time span is 293 or more years. # https://github.com/klindsay28/CESM2_coup_carb_cycle_JAMES/issues/7 if tb.dtype == np.dtype("O"): units = "days since 0001-01-01" calendar = "noleap" tb_vals = cftime.date2num(ds[tb_name].values, units=units, calendar=calendar) tb_mid_decode = cftime.num2date( tb_vals.mean(axis=1), units=units, calendar=calendar ) ds_out[time_name] = ds[time_name].copy(data=tb_mid_decode) else: ds_out[time_name] = ds[time_name].copy(data=tb.mean(bounds_dim)) return ds_out
def make_netcdf(config, in_file): """ Main program function """ out_file = f"{os.path.splitext(in_file)[0]}.nc" quince = pd.read_csv(in_file, parse_dates=["Date/Time"], low_memory=False) with Dataset(out_file, "w", format="NETCDF4") as nc: # Set global attributes from config for attr in config["global_attributes"]: nc.setncattr(attr, config["global_attributes"][attr]) nc.setncattr("creation_date", datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')) nc.createDimension("time") timevar = nc.createVariable("time", "f8", ("time", )) timevar.units = "days since 1970-01-01 00:00:00.0" timevar[:] = date2num(quince["Date/Time"], timevar.units) for column in config["columns"]: column_info = config["columns"][column] var = nc.createVariable(column_info["variable"], column_info["type"], ("time", )) var.units = column_info["units"] var[:] = quince[column]
def test_decode_non_standard_calendar_single_element(calendar): cftime = _import_cftime() units = 'days since 0001-01-01' try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: # Must be using the standalone cftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) actual = coding.times.decode_cf_datetime(num_time, units, calendar=calendar) if cftime.__name__ == 'cftime': expected = np.asarray( cftime.num2date(num_time, units, calendar, only_use_cftime_datetimes=True)) else: expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual
def cfencode_time(dsvar, calendar="proleptic_gregorian", units="seconds since 1970-01-01 00:00:00"): dsvar[:] = date2num(dsvar[:], units, calendar) dsvar.attrs["units"] = units dsvar.attrs["calendar"] = calendar dsvar.attrs["axis"] = "T"
def test_decode_non_standard_calendar_inside_timestamp_range( calendar, enable_cftimeindex): if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') noleap_time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) if enable_cftimeindex: expected = cftime.num2date(noleap_time, units, calendar=calendar) expected_dtype = np.dtype('O') else: expected = times.values expected_dtype = np.dtype('M8[ns]') with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( noleap_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all()
def test_decode_dates_outside_timestamp_range( calendar, enable_cftimeindex): from datetime import datetime if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' times = [datetime(1, 4, 1, h) for h in range(1, 5)] noleap_time = cftime.date2num(times, units, calendar=calendar) if enable_cftimeindex: expected = cftime.num2date(noleap_time, units, calendar=calendar, only_use_cftime_datetimes=True) else: expected = cftime.num2date(noleap_time, units, calendar=calendar) expected_date_type = type(expected[0]) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( noleap_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all()
def get_masked_datetime_array(t, tvar, mask_nan=True): # If we are passed in a scalar... return a scalar if isinstance(t, np.ma.core.MaskedConstant): return t elif np.isscalar(t): return num2date(t, tvar.units, getattr(tvar, 'calendar', 'standard')) if mask_nan is True: t = np.ma.masked_invalid(t) t_cal = getattr(tvar, 'calendar', 'standard') # Get the min value we can have and mask anything else # This is limited by **python** datetime objects and not # nc4 objects. The min nc4 datetime object is # min_date = nc4.netcdftime.datetime(-4713, 1, 1, 12, 0, 0, 40) # There is no max date for nc4. min_nums = date2num([datetime.min, datetime.max], tvar.units, t_cal) t = np.ma.masked_outside(t, *min_nums) dts = num2pydate(t, tvar.units, t_cal) if isinstance(dts, (datetime, cfdt)): dts = np.array([dts.isoformat()], dtype='datetime64') return dts
def get_time_bounds(self, time_coord): """ Get contiguous time bounds for sea ice maps Creates new time bounds [ [01-01-current year, 01-01-next year], ] """ dt_object = cftime.num2pydate(time_coord.points[0], time_coord.units.name) start = datetime.datetime(dt_object.year, 1, 1) end = datetime.datetime(start.year + 1, 1, 1) start_seconds = cftime.date2num(start, time_coord.units.name) end_seconds = cftime.date2num(end, time_coord.units.name) new_bounds = np.array([[start_seconds, end_seconds]]) return new_bounds
def _verif_dates_xr(hindcast, alignment, reference, date2num_units): """Create ``valid_time`` ``xr.DataArray`` with dims lead and init in units passed to cftime.date2num.""" inits, verif_dates = return_inits_and_verif_dates( hindcast.get_initialized().rename({"init": "time"}), hindcast.get_observations(), alignment, reference=reference, hist=hindcast.get_uninitialized() if isinstance( hindcast.get_uninitialized(), xr.Dataset) else None, ) verif_dates_xr = xr.concat( [ xr.DataArray( cftime.date2num(verif_dates[k], date2num_units), dims="init", coords={"init": v.rename({ "time": "init" }).to_index()}, name="valid_time", attrs=dict(units=date2num_units), ) for k, v in inits.items() ], dim="lead", ).assign_coords(lead=hindcast.get_initialized().lead) return verif_dates_xr
def test_decode_non_standard_calendar_inside_timestamp_range( calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') non_standard_time = cftime.date2num( times.to_pydatetime(), units, calendar=calendar) if cftime.__name__ == 'cftime': expected = cftime.num2date( non_standard_time, units, calendar=calendar, only_use_cftime_datetimes=True) else: expected = cftime.num2date(non_standard_time, units, calendar=calendar) expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime( non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all()
def st2rt(array, units_in, units_out, dummy1=None): """The returned array is always independent. :Parameters: array: numpy array-like of ISO 8601 date-time strings units_in: `Units` or `None` units_out: `Units` dummy1: Ignored. :Returns: `numpy.ndarray` An array of floats with the same shape as *array*. """ array = st2dt(array, units_in) # array = units_out._utime.date2num(array) array = cftime.date2num(array, units=units_out.units, calendar=units_out._utime.calendar) if not np.ndim(array): array = np.asanyarray(array) return array
def test_decode_dates_outside_timestamp_range(calendar): from datetime import datetime import cftime units = "days since 0001-01-01" times = [datetime(1, 4, 1, h) for h in range(1, 5)] time = cftime.date2num(times, units, calendar=calendar) expected = cftime.num2date(time, units, calendar=calendar, only_use_cftime_datetimes=True) expected_date_type = type(expected[0]) with warnings.catch_warnings(): warnings.filterwarnings("ignore", "Unable to decode time axis") actual = coding.times.decode_cf_datetime(time, units, calendar=calendar) assert all(isinstance(value, expected_date_type) for value in actual) abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, "s")).all()
def set_time(self, time_coord_name=None, year_offset=None): """store the original time and time_bound data from the dataset; ensure that time_bound, if present, is not decoded. """ self.infer_time_coord_name(time_coord_name) self.year_offset = year_offset self.time = self._ds[self.time_coord_name].copy() self.time_orig_decoded = self.isdecoded(self.time) self.infer_time_bound_var() if self.tb_name is None: self.time_bound = None else: self.time_bound = self._ds[self.tb_name].copy() if self.isdecoded(self._ds[self.tb_name]): try: tb_data = cftime.date2num( self._ds[self.tb_name], units=self.time_attrs['units'], calendar=self.time_attrs['calendar'], ) self.time_bound.data = tb_data except Exception as exc: print('Please open dataset with `decode_times=False`') raise exc self.setup() return self
def time_set_mid(ds, time_name): """ set ds[time_name] to midpoint of ds[time_name].attrs['bounds'], if bounds attribute exists type of ds[time_name] is not changed ds is returned """ if 'bounds' not in ds[time_name].attrs: return ds # determine units and calendar of unencoded time values if ds[time_name].dtype == np.dtype('O'): units = 'days since 0000-01-01' calendar = 'noleap' else: units = ds[time_name].attrs['units'] calendar = ds[time_name].attrs['calendar'] # construct unencoded midpoint values, assumes bounds dim is 2nd tb_name = ds[time_name].attrs['bounds'] if ds[tb_name].dtype == np.dtype('O'): tb_vals = cftime.date2num(ds[tb_name].values, units=units, calendar=calendar) else: tb_vals = ds[tb_name].values tb_mid = tb_vals.mean(axis=1) # set ds[time_name] to tb_mid if ds[time_name].dtype == np.dtype('O'): ds[time_name] = cftime.num2date(tb_mid, units=units, calendar=calendar) else: ds[time_name] = tb_mid return ds
def test_decode_non_standard_calendar_inside_timestamp_range(calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times = pd.date_range('2001-04-01-00', end='2001-04-30-23', freq='H') non_standard_time = cftime.date2num(times.to_pydatetime(), units, calendar=calendar) if cftime.__name__ == 'cftime': expected = cftime.num2date(non_standard_time, units, calendar=calendar, only_use_cftime_datetimes=True) else: expected = cftime.num2date(non_standard_time, units, calendar=calendar) expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime(non_standard_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff = abs(actual - expected) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff <= np.timedelta64(1, 's')).all()
def dataList2NetCDF( self, ncFileName, shortVarNameList, varFieldList, timeStamp, posCnt=None ): rootgrp = nc.Dataset(ncFileName, "a") date_time = rootgrp.variables["time"] if posCnt == None: posCnt = len(date_time) for shortVarName in shortVarNameList: date_time[posCnt] = cftime.date2num( timeStamp, date_time.units, date_time.calendar ) varField = varFieldList[shortVarName] # flip variable if necessary (to follow cf_convention) if self.netcdf_y_orientation_follow_cf_convention: varField = np.flipud(varField) rootgrp.variables[shortVarName][posCnt, :, :] = varField rootgrp.sync() rootgrp.close()
def dataList2NetCDF(self, ncFileName, shortVarNameList, varFieldList, timeStamp, posCnt=None): rootgrp = nc.Dataset(ncFileName, "a") date_time = rootgrp.variables["time"] if posCnt == None: posCnt = len(date_time) for shortVarName in shortVarNameList: date_time[posCnt] = cftime.date2num(timeStamp, date_time.units, date_time.calendar) varField = varFieldList[shortVarName] # flip variable if necessary (to follow cf_convention) if self.netcdf_y_orientation_follow_cf_convention: varField = np.flipud(varField) rootgrp.variables[shortVarName][posCnt, :, :] = varField rootgrp.sync() rootgrp.close()
def open_ann_fosi(anomaly=True): #-- open the dataset xr_open_ds = { #'chunks' : {'time':1}, # chunking breaks "rolling" method 'decode_coords': False, 'decode_times': False } case = 'g.e11_LENS.GECOIAF.T62_g16.009' file_in = f'/glade/work/yeager/{case}/budget_O2_npac_{case}.0249-0316.nc' ds = xr.open_dataset(file_in, **xr_open_ds) #-- convert units ds = conform_budget_dataset(ds) grid = ds.drop([v for v in ds.variables if 'time' in ds[v].dims]) #-- interpret time: make time into "year" offset = cftime.date2num(cftime.DatetimeGregorian(1699, 1, 1), ds.time.attrs['units'], ds.time.attrs['calendar']) ds['date'] = cftime.num2date(ds.time + offset, ds.time.attrs['units'], ds.time.attrs['calendar']) ds.time.values = [d.year * 1. for d in ds.date.values] #-- make into an anomaly if anomaly: for v in ds.variables: if 'time' in ds[v].dims and v != 'time': attrs = ds[v].attrs ds[v] = ds[v] - ds[v].sel(time=year_range_clim).mean('time') ds[v].attrs = attrs return ds
def test_decode_non_standard_calendar_single_element_fallback( calendar, enable_cftimeindex): if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: # Must be using standalone netcdftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) if enable_cftimeindex: actual = coding.times.decode_cf_datetime( num_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) else: with pytest.warns(SerializationWarning, match='Unable to decode time axis'): actual = coding.times.decode_cf_datetime( num_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual
def datetime_to_float(x): """ converts datetime values to float for regression """ time_ = x.time.values if type(x.time.values[0])==np.datetime64: x.time.values = np.array(x.time.values - np.datetime64('0001-01-01'), dtype=np.float) elif type(x.time.values[0])==cftime._cftime.Datetime360Day: x.time.values = cftime.date2num(x.time.values, units='months since 0001-01-01', calendar='360_day') return x, time_
def datetime_from_grid(grid, epoch=False, **kwargs): """ Return a datetime for the volume start in a Grid. """ if epoch: dtrad = num2date(grid.time['data'][0], grid.time['units']) epnum = date2num(dtrad, EPOCH_UNITS) return num2date(epnum, EPOCH_UNITS, **kwargs) else: return num2date(grid.time['data'][0], grid.time['units'], **kwargs)
def datetimes_from_radar(radar, epoch=False, **kwargs): """ Return an array of datetimes for the rays in a Radar. """ if epoch: dtrad = num2date(radar.time['data'][:], radar.time['units']) epnum = date2num(dtrad, EPOCH_UNITS) return num2date(epnum, EPOCH_UNITS, **kwargs) else: return num2date(radar.time['data'][:], radar.time['units'], **kwargs)
def save_to_netcdf(self, r_text, file_path): results = re.findall( r'^(\d+-\d+-\d+ \d+:\d+:\d+.\d+)\s*(\d+)\s*([+\-\d.]+)\s*([+\-\d.]+)\s*([+\-\d.]+)\s*([+\-\d.]+)', r_text, re.M) results = list(zip(*results)) # time_array = np.array([(datetime.datetime.strptime(dtstr+'000', "%Y-%m-%d %H:%M:%S.%f") # - datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1) # for dtstr in results[0]]) dts = [ datetime.datetime.strptime(dtstr + '000', "%Y-%m-%d %H:%M:%S.%f") for dtstr in results[0] ] time_array = np.array( cftime.date2num(dts, units='seconds since 1970-01-01 00:00:00.0')) print('From {} to {}.'.format( datetime.datetime.utcfromtimestamp(time_array[0]), datetime.datetime.utcfromtimestamp(time_array[-1]))) asy_d_array = np.array(results[2]) asy_d_array.astype(np.float32) asy_h_array = np.array(results[3]) asy_h_array.astype(np.float32) sym_d_array = np.array(results[4]) sym_d_array.astype(np.float32) sym_h_array = np.array(results[5]) sym_h_array.astype(np.float32) num_rows = len(results[0]) fnc = netCDF4.Dataset(file_path, 'w') fnc.createDimension('UNIX_TIME', num_rows) fnc.title = "WDC ASY/SYM indices" time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', )) time.units = 'seconds since 1970-01-01 00:00:00.0' asy_d = fnc.createVariable('ASY_D', np.float32, ('UNIX_TIME', )) asy_h = fnc.createVariable('ASY_H', np.float32, ('UNIX_TIME', )) sym_d = fnc.createVariable('SYM_D', np.float32, ('UNIX_TIME', )) sym_h = fnc.createVariable('SYM_H', np.float32, ('UNIX_TIME', )) time[::] = time_array[::] asy_d[::] = asy_d_array[::] asy_h[::] = asy_h_array[::] sym_d[::] = sym_d_array[::] sym_h[::] = sym_h_array[::] # for i, res in enumerate(results): # dt = datetime.datetime.strptime(res[0]+'000', "%Y-%m-%d %H:%M:%S.%f") # time[i] = (dt - datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1) # asy_d[i] = float(res[2]) # asy_h[i] = float(res[3]) # sym_d[i] = float(res[4]) # sym_h[i] = float(res[5]) fnc.close() mylog.StreamLogger.info( "The requested data has been downloaded and saved in the file {}.". format(file_path)) self.done = True
def _make_index(time): year = int(time.dt.year[0]) doys = cftime.date2num(time, f"days since {year:04d}-01-01", calendar=calendar) return DataArray( year + doys / _days_in_year(year, calendar), dims=(dim,), coords=time.coords, name=dim, )
def save_to_netcdf(self, ystr, file_path): with open(file_path, 'r') as f: text = f.read() results = re.findall( r'^(\d{4} \d{2} \d{2})\s*([-+]?\d*\.\d+|\d+)' + r'\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)' + r'\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)\s*([-+]?\d*\.\d+|\d+)', text, re.M) results = list(zip(*results)) dts = [ datetime.datetime.strptime(dtstr, "%Y %m %d") + datetime.timedelta(hours=float(hs)) for dtstr, hs in zip(results[0], results[2]) ] time_array = np.array( cftime.date2num(dts, units='seconds since 1970-01-01 00:00:00.0')) hp_array = np.array(results[5]).astype(np.float32) hp_array = np.where(hp_array == -1, np.nan, hp_array) ap_array = np.array(results[6]).astype(np.float32) ap_array = np.where(ap_array == -1, np.nan, ap_array) flag_array = np.array(results[7]).astype(np.int32) num_rows = len(results[0]) fp = file_path.parent.resolve() / (f"GFZ_Hpo_Hp{self.data_res}_" + ystr + '.nc') fp.parent.resolve().mkdir(parents=True, exist_ok=True) fnc = nc.Dataset(fp, 'w') fnc.createDimension('UNIX_TIME', num_rows) fnc.title = f"GFZ Hpo/Hp{self.data_res} index" time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', )) time.units = 'seconds since 1970-01-01 00:00:00.0' hp = fnc.createVariable('Hp', np.float32, ('UNIX_TIME', )) ap = fnc.createVariable('ap', np.float32, ('UNIX_TIME', )) flag = fnc.createVariable('FLAG', np.float32, ('UNIX_TIME', )) time[::] = time_array[::] hp[::] = hp_array[::] ap[::] = ap_array[::] flag[::] = flag_array print('From {} to {}.'.format( datetime.datetime.utcfromtimestamp(time_array[0]), datetime.datetime.utcfromtimestamp(time_array[-1]))) mylog.StreamLogger.info( "The requested Hpo/Hp{} data has been downloaded and saved in the file {}." .format(self.data_res, fp)) fnc.close() self.done = True
def setup_module(module): if verbose: print ("setup_module module:%s" % module.__name__) if verbose: print ("Python version: {}".format(sys.version)) # Put any setup code in here, like making temporary files # Make 5 years of a noleap calendar on the first of each month global times for y in range(1,6): for m in range(1,13): times.append(np.round(cftime.date2num(cftime.datetime(y,m,1),units='days since 01-01-01',calendar='noleap'),8)) times = np.array(times)
def datetimes_from_dataset(dataset, epoch=False, **kwargs): """ Return an array of datetimes for the times in a netCDF Dataset. """ if epoch: dtdata = num2date(dataset.variables['time'][:], dataset.variables['time'].units) epnum = date2num(dtdata, EPOCH_UNITS) return num2date(epnum, EPOCH_UNITS, **kwargs) else: return num2date(dataset.variables['time'][:], dataset.variables['time'].units, **kwargs)
def test_decode_multidim_time_outside_timestamp_range( calendar, enable_cftimeindex): from datetime import datetime if enable_cftimeindex: pytest.importorskip('cftime') cftime = _import_cftime() units = 'days since 0001-01-01' times1 = [datetime(1, 4, day) for day in range(1, 6)] times2 = [datetime(1, 5, day) for day in range(1, 6)] noleap_time1 = cftime.date2num(times1, units, calendar=calendar) noleap_time2 = cftime.date2num(times2, units, calendar=calendar) mdim_time = np.empty((len(noleap_time1), 2), ) mdim_time[:, 0] = noleap_time1 mdim_time[:, 1] = noleap_time2 if enable_cftimeindex: expected1 = cftime.num2date(noleap_time1, units, calendar, only_use_cftime_datetimes=True) expected2 = cftime.num2date(noleap_time2, units, calendar, only_use_cftime_datetimes=True) else: expected1 = cftime.num2date(noleap_time1, units, calendar) expected2 = cftime.num2date(noleap_time2, units, calendar) with warnings.catch_warnings(): warnings.filterwarnings('ignore', 'Unable to decode time axis') actual = coding.times.decode_cf_datetime( mdim_time, units, calendar=calendar, enable_cftimeindex=enable_cftimeindex) assert actual.dtype == np.dtype('O') abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, 's')).all() assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def test_decode_nonstandard_calendar_multidim_time_inside_timestamp_range( calendar): cftime = _import_cftime() units = 'days since 0001-01-01' times1 = pd.date_range('2001-04-01', end='2001-04-05', freq='D') times2 = pd.date_range('2001-05-01', end='2001-05-05', freq='D') time1 = cftime.date2num(times1.to_pydatetime(), units, calendar=calendar) time2 = cftime.date2num(times2.to_pydatetime(), units, calendar=calendar) mdim_time = np.empty((len(time1), 2), ) mdim_time[:, 0] = time1 mdim_time[:, 1] = time2 if cftime.__name__ == 'cftime': expected1 = cftime.num2date(time1, units, calendar, only_use_cftime_datetimes=True) expected2 = cftime.num2date(time2, units, calendar, only_use_cftime_datetimes=True) else: expected1 = cftime.num2date(time1, units, calendar) expected2 = cftime.num2date(time2, units, calendar) expected_dtype = np.dtype('O') actual = coding.times.decode_cf_datetime( mdim_time, units, calendar=calendar) assert actual.dtype == expected_dtype abs_diff1 = abs(actual[:, 0] - expected1) abs_diff2 = abs(actual[:, 1] - expected2) # once we no longer support versions of netCDF4 older than 1.1.5, # we could do this check with near microsecond accuracy: # https://github.com/Unidata/netcdf4-python/issues/355 assert (abs_diff1 <= np.timedelta64(1, 's')).all() assert (abs_diff2 <= np.timedelta64(1, 's')).all()
def prepare_nc( trg_file, times, x, y, metadata={}, logging=logging, units="Days since 1900-01-01 00:00:00", calendar="gregorian", ): """ This function prepares a NetCDF file with given metadata, for a certain year, daily basis data The function assumes a gregorian calendar and a time unit 'Days since 1900-01-01 00:00:00' """ logger.info('Setting up "' + trg_file + '"') times_list = cftime.date2num(times, units=units, calendar=calendar) nc_trg = nc.Dataset(trg_file, "w") logger.info("Setting up dimensions and attributes") nc_trg.createDimension("time", 0) # NrOfDays*8 nc_trg.createDimension("lat", len(y)) nc_trg.createDimension("lon", len(x)) times_nc = nc_trg.createVariable("time", "f8", ("time",)) times_nc.units = units times_nc.calendar = calendar times_nc.standard_name = "time" times_nc.long_name = "time" times_nc[:] = times_list y_var = nc_trg.createVariable("lat", "f4", ("lat",)) y_var.standard_name = "latitude" y_var.long_name = "latitude" y_var.units = "degrees_north" x_var = nc_trg.createVariable("lon", "f4", ("lon",)) x_var.standard_name = "longitude" x_var.long_name = "longitude" x_var.units = "degrees_east" y_var[:] = y x_var[:] = x projection = nc_trg.createVariable("projection", "c") projection.long_name = "wgs84" projection.EPSG_code = "EPSG:4326" projection.proj4_params = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs" projection.grid_mapping_name = "latitude_longitude" # now add all attributes from user-defined metadata for attr in metadata: nc_trg.setncattr(attr, metadata[attr]) nc_trg.sync() return nc_trg
def test_decode_non_standard_calendar_single_element( calendar): cftime = _import_cftime() units = 'days since 0001-01-01' try: dt = cftime.netcdftime.datetime(2001, 2, 29) except AttributeError: # Must be using the standalone cftime library dt = cftime.datetime(2001, 2, 29) num_time = cftime.date2num(dt, units, calendar) actual = coding.times.decode_cf_datetime( num_time, units, calendar=calendar) if cftime.__name__ == 'cftime': expected = np.asarray(cftime.num2date( num_time, units, calendar, only_use_cftime_datetimes=True)) else: expected = np.asarray(cftime.num2date(num_time, units, calendar)) assert actual.dtype == np.dtype('O') assert expected == actual
def prepare_nc( trgFile, timeList, x, y, metadata, logger, EPSG="EPSG:4326", units=None, calendar="gregorian", Format="NETCDF4", complevel=9, zlib=True, least_significant_digit=None, FillValue=1e31, ): """ This function prepares a NetCDF file with given metadata, for a certain year, daily basis data The function assumes a gregorian calendar and a time unit 'Days since 1900-01-01 00:00:00' """ logger.info("Setting up netcdf output: " + trgFile) if units == None: # Use start of the run epoch = timeList[0] units = "seconds since %04d-%02d-%02d %02d:%02d:%02d.0 00:00" % ( epoch.year, epoch.month, epoch.day, epoch.hour, epoch.minute, epoch.second, ) startDayNr = cftime.date2num( timeList[0].replace(tzinfo=None), units=units, calendar=calendar ) endDayNr = cftime.date2num( timeList[-1].replace(tzinfo=None), units=units, calendar=calendar ) timeAR = np.linspace(startDayNr, endDayNr, num=len(timeList)) if os.path.exists(trgFile): os.remove(trgFile) nc_trg = netCDF4.Dataset( trgFile, "w", format=Format, zlib=zlib, complevel=complevel ) logger.info( "Setting up dimensions and attributes. Steps: " + str(len(timeList)) + " lat: " + str(len(y)) + " lon: " + str(len(x)) ) if len(timeAR) == 1: nc_trg.createDimension("time", 1) else: nc_trg.createDimension("time", 0) # NrOfDays*8 DateHour = nc_trg.createVariable( "time", "f8", ("time",), fill_value=FillValue, zlib=zlib, complevel=complevel ) DateHour.units = units DateHour.calendar = calendar DateHour.standard_name = "time" DateHour.long_name = "time" DateHour.axis = "T" DateHour[:] = timeAR # make a proj4 string srs = osgeo.osr.SpatialReference() res = srs.ImportFromEPSG(int(EPSG[5:])) if res != 0: logger.error( "EPGS not converted correctly: " + EPSG + ". Is the GDAL_DATA environment variable set correctly?" ) sys.exit(1) projStr = srs.ExportToProj4() proj_src = "+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs" if srs.IsProjected() == 0: # ONly lat lon needed nc_trg.createDimension("lat", len(y)) nc_trg.createDimension("lon", len(x)) y_var = nc_trg.createVariable( "lat", "f4", ("lat",), fill_value=FillValue, zlib=zlib, complevel=complevel ) y_var.standard_name = "latitude" y_var.long_name = "latitude" y_var.units = "degrees_north" y_var.axis = "Y" x_var = nc_trg.createVariable( "lon", "f4", ("lon",), fill_value=FillValue, zlib=zlib, complevel=complevel ) x_var.standard_name = "longitude" x_var.long_name = "longitude" x_var.units = "degrees_east" x_var.axis = "X" y_var[:] = y x_var[:] = x crs = nc_trg.createVariable("crs", "c") crs.long_name = "wgs84" crs.proj4_params = "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs" crs.grid_mapping_name = "latitude_longitude" else: # Assume regular grid in m nc_trg.createDimension("y", len(y)) nc_trg.createDimension("x", len(x)) y_var = nc_trg.createVariable( "y", "f4", ("y",), fill_value=FillValue, zlib=zlib, complevel=complevel ) y_var.standard_name = "projection_y_coordinate" y_var.long_name = "y-coordinate in Cartesian system" y_var.units = "m" y_var.axis = "Y" x_var = nc_trg.createVariable( "x", "f4", ("x",), fill_value=FillValue, zlib=zlib, complevel=complevel ) x_var.standard_name = "projection_x_coordinate" x_var.long_name = "x-coordinate in Cartesian system" x_var.units = "m" x_var.axis = "X" y_var[:] = y x_var[:] = x crs = nc_trg.createVariable("crs", "c") crs.long_name = EPSG crs.grid_mapping_name = "universal_transverse_mercator" crs.utm_zone_number = srs.GetUTMZone() crs.semi_major_axis = srs.GetSemiMajor() crs.inverse_flattening = srs.GetInvFlattening() crs._CoordinateTransformType = "Projection" crs._CoordinateAxisTypes = "y x" crs.proj4_params = projStr # Also write lat lon fields XI, YI = np.meshgrid(x, y) lon_vals, lat_vals = convertCoord(projStr, proj_src, XI, YI) # Need to create lat-lon fields lat = nc_trg.createVariable("lat", "f4", ("y", "x")) lat.standard_name = "latitude" lat.long_name = "latitude coordinate" lat.units = "degrees_north" lat.coordinates = "lat lon" lat.grid_mapping = "wgs84" # lat._CoordinateAxisType = "Lat" lat[:, :] = lat_vals lon = nc_trg.createVariable("lon", "f4", ("y", "x")) lon.standard_name = "longitude" lon.long_name = "longitude coordinate" lon.units = "degrees_east" lon.coordinates = "lat lon" lon.grid_mapping = "wgs84" # lon._CoordinateAxisType = "Lon" lon[:, :] = lon_vals crs.EPSG_code = EPSG # now add all attributes from user-defined metadata for attr in metadata: nc_trg.setncattr(attr, metadata[attr]) nc_trg.sync() nc_trg.close()
def encode_datetime(d): return np.nan if d is None else cftime.date2num(d, units, calendar)