def __init__(
self,
filename,
product,
parameter=["swvl1", "swvl2"],
subgrid=None,
mask_seapoints=False,
array_1D=False,
mode="r",
):
super(ERANcImg, self).__init__(filename, mode=mode)
if type(parameter) == str:
parameter = [parameter]
# look up short names
self.parameter = lookup(product, parameter)["short_name"].values
self.mask_seapoints = mask_seapoints
self.array_1D = array_1D
self.subgrid = subgrid
if self.subgrid and not self.array_1D:
warnings.warn(
"Reading spatial subsets as 2D arrays ony works if there "
"is an equal number of points in each line")
def __init__(self,
filename,
product,
parameter=['swvl1', 'swvl2'],
mode='r',
subgrid=None,
mask_seapoints=False,
array_1D=False):
super(ERANcImg, self).__init__(filename, mode=mode)
if type(parameter) == str:
parameter = [parameter]
# look up short names
self.parameter = lookup(product, parameter)['short_name'].values
self.mask_seapoints = mask_seapoints
self.array_1D = array_1D
self.subgrid = subgrid
def __init__(
self,
filename,
product,
parameter=("swvl1", "swvl2"),
subgrid=None,
mask_seapoints=False,
array_1D=True,
mode="r",
):
super(ERAGrbImg, self).__init__(filename, mode=mode)
if type(parameter) == str:
parameter = [parameter]
self.parameter = lookup(
product, parameter)["short_name"].values # look up short names
self.product = product
self.mask_seapoints = mask_seapoints
self.array_1D = array_1D
self.subgrid = subgrid
def read(self, timestamp=None):
"""
Read data from the loaded image file.
Parameters
---------
timestamp : datetime, optional (default: None)
Specific date (time) to read the data for.
"""
grbs = pygrib.open(self.filename)
grid = self.subgrid
return_img = {}
return_metadata = {}
var_msg_lut = {p: None for p in self.parameter}
sea_mask = None
for N in range(grbs.messages):
n = N + 1
message = grbs.message(n)
param_name = str(message.cfVarNameECMF)
if param_name == "lsm":
if self.mask_seapoints and sea_mask is None:
sea_mask = message.values.flatten()
if param_name not in self.parameter:
continue
else:
var_msg_lut[param_name] = n
# available variables
shape = None
for param_name, n in var_msg_lut.items():
if n is None:
continue
return_metadata[param_name] = {}
message = grbs.message(n)
param_data = message.values.flatten()
if not shape:
shape = param_data.shape
return_img[param_name] = param_data
if grid is None:
lats, lons = message.latlons()
try:
res_lat, res_lon = get_grid_resolution(lats, lons)
grid = ERA_RegularImgGrid(res_lat, res_lon)
except ValueError: # when grid not regular
lons_gt_180 = np.where(lons > 180.0)
lons[lons_gt_180] = lons[lons_gt_180] - 360
grid = ERA_IrregularImgGrid(lons, lats)
return_metadata[param_name]["units"] = message["units"]
return_metadata[param_name]["long_name"] = \
message["parameterName"]
if "levels" in message.keys():
return_metadata[param_name]["depth"] = "{:} cm".format(
message["levels"])
if self.mask_seapoints:
if sea_mask is None:
raise IOError(
"No land sea mask parameter (lsm) in passed image"
" for masking.")
else:
# mask the loaded data
for name in return_img.keys():
param_data = return_img[name]
param_data = np.ma.array(
param_data,
mask=np.logical_not(sea_mask),
fill_value=np.nan,
)
param_data = param_data.filled()
return_img[name] = param_data
grbs.close()
# missing variables
for param_name, n in var_msg_lut.items():
if n is not None:
continue
param_data = np.full(shape, np.nan)
warnings.warn("Cannot load variable {var} from file {thefile}. "
"Filling image with NaNs.".format(
var=param_name, thefile=self.filename))
return_img[param_name] = param_data
return_metadata[param_name] = {}
return_metadata[param_name]["long_name"] = lookup(
self.product, [param_name]).iloc[0]["long_name"]
if self.array_1D:
return Image(
grid.activearrlon,
grid.activearrlat,
return_img,
return_metadata,
timestamp,
)
else:
nlat = np.unique(grid.activearrlat).size
nlon = np.unique(grid.activearrlon).size
for key in return_img:
return_img[key] = return_img[key].reshape((nlat, nlon))
return Image(
grid.activearrlon.reshape(nlat, nlon),
grid.activearrlat.reshape(nlat, nlon),
return_img,
return_metadata,
timestamp,
)
def download_and_move(
target_path,
startdate,
enddate,
product="era5",
variables=None,
keep_original=False,
h_steps=[0, 6, 12, 18],
grb=False,
dry_run=False,
grid=None,
remap_method="bil",
cds_kwds={},
stepsize="month",
) -> int:
"""
Downloads the data from the ECMWF servers and moves them to the target
path.
This is done in 30 day increments between start and end date.
The files are then extracted into separate grib files per parameter and
stored in yearly folders under the target_path.
Parameters
----------
target_path : str
Path where the files are stored to
startdate: datetime
first date to download
enddate: datetime
last date to download
product : str, optional (default: ERA5)
Either ERA5 or ERA5Land
variables : list, optional (default: None)
Name of variables to download
keep_original: bool
keep the original downloaded data
h_steps: list
List of full hours to download data at the selected dates e.g [0, 12]
grb: bool, optional (default: False)
Download data as grib files
dry_run: bool
Do not download anything, this is just used for testing the functions
grid : dict, optional
A grid on which to remap the data using CDO. This must be a dictionary
using CDO's grid description format, e.g.::
grid = {
"gridtype": "lonlat",
"xsize": 720,
"ysize": 360,
"xfirst": -179.75,
"yfirst": 89.75,
"xinc": 0.5,
"yinc": -0.5,
}
Default is to use no regridding.
remap_method : str, optional
Method to be used for regridding. Available methods are:
- "bil": bilinear (default)
- "bic": bicubic
- "nn": nearest neighbour
- "dis": distance weighted
- "con": 1st order conservative remapping
- "con2": 2nd order conservative remapping
- "laf": largest area fraction remapping
cds_kwds: dict, optional
Additional arguments to be passed to the CDS API retrieve request.
stepsize : str, optional
Size of steps for requests, can be "month" or "day".
Returns
-------
status_code: int
0 : Downloaded data ok
-1 : Error
-10 : No data available for requested time period
"""
product = product.lower()
if variables is None:
variables = default_variables(product=product)
else:
# find the dl_names
variables = lookup(name=product, variables=variables)
variables = variables["dl_name"].values.tolist()
curr_start = startdate
if dry_run:
warnings.warn("Dry run does not create connection to CDS")
c = None
cds_status_tracker = None
else:
cds_status_tracker = CDSStatusTracker()
c = cdsapi.Client(
error_callback=cds_status_tracker.handle_error_function)
pool = multiprocessing.Pool(1)
while curr_start <= enddate:
status_code = -1
sy, sm, sd = curr_start.year, curr_start.month, curr_start.day
y, m = sy, sm
if stepsize == "month":
sm_days = calendar.monthrange(sy,
sm)[1] # days in the current month
if (enddate.year == y) and (enddate.month == m):
d = enddate.day
else:
d = sm_days
elif stepsize == "day":
d = sd
else:
raise ValueError(f"Invalid stepsize: {stepsize}")
curr_end = datetime(y, m, d)
fname = "{start}_{end}.{ext}".format(
start=curr_start.strftime("%Y%m%d"),
end=curr_end.strftime("%Y%m%d"),
ext="grb" if grb else "nc",
)
downloaded_data_path = os.path.join(target_path, "temp_downloaded")
if not os.path.exists(downloaded_data_path):
os.mkdir(downloaded_data_path)
dl_file = os.path.join(downloaded_data_path, fname)
finished, i = False, 0
while (not finished) and (i < 5): # try max 5 times
try:
finished = download_era5(
c,
years=[sy],
months=[sm],
days=range(sd, d + 1),
h_steps=h_steps,
variables=variables,
grb=grb,
product=product,
target=dl_file,
dry_run=dry_run,
cds_kwds=cds_kwds,
)
status_code = 0
break
except: # noqa: E722
# If no data is available we don't need to retry
if (cds_status_tracker.download_statuscode ==
CDSStatusTracker.statuscode_unavailable):
status_code = -10
break
# delete the partly downloaded data and retry
if os.path.isfile(dl_file):
os.remove(dl_file)
finished = False
i += 1
continue
if status_code == 0:
if grb:
pool.apply_async(
save_gribs_from_grib,
args=(dl_file, target_path),
kwds=dict(
product_name=product.upper(),
keep_original=keep_original,
),
)
else:
pool.apply_async(
save_ncs_from_nc,
args=(
dl_file,
target_path,
),
kwds=dict(
product_name=product.upper(),
grid=grid,
remap_method=remap_method,
keep_original=keep_original,
),
)
curr_start = curr_end + timedelta(days=1)
pool.close()
pool.join()
# remove temporary files
if not keep_original:
shutil.rmtree(downloaded_data_path)
if grid is not None:
gridpath = os.path.join(target_path, "grid.txt")
if os.path.exists(gridpath):
os.unlink(gridpath)
weightspath = os.path.join(target_path, "remap_weights.nc")
if os.path.exists(weightspath):
os.unlink(weightspath)
return status_code
def download_and_move(
target_path,
startdate,
enddate,
variables=None,
keep_original=False,
grid_size=None,
type="an",
h_steps=(0, 6, 12, 18),
steps=(0, ),
grb=False,
dry_run=False,
):
"""
Downloads the data from the ECMWF servers and moves them to the target
path. This is done in 30 days increments between start and end date to
be efficient with the MARS system.
See the recommendation for doing it this way in
https://software.ecmwf.int/wiki/display/WEBAPI/ERA-Interim+daily+retrieval+efficiency
The files are then extracted into separate grib/nc files and stored in
yearly folders under the target_path.
Parameters
----------
target_path: str
Path to which to copy the extracted parameter files
startdate: datetime
First date to download
enddate: datetime
Last date to download
variables : list, optional (default: None)
List of variable ids to pass to the client, if None are passed,
the default variable ids will be downloaded.
keep_original: bool, optional (default: False)
Keep the original downloaded data
grid_size: list, optional (default: None)
[lon, lat] extent of the grid (regular for netcdf, at lat=0 for grib)
If None is passed, the default grid size for the data product is used.
type : str, optional (default: 'an')
Data stream, model to download data for (fc=forecase)
h_steps: list, optional (default: [0, 6, 12, 18])
List of full hours to download data at the selected dates
grb: bool, optional (default: False)
Download data as grib files instead of netcdf files
dry_run: bool
Do not download anything, this is just used for testing the functions
"""
product = "eraint"
if variables is None:
variables = default_variables()
else:
# find the dl_names
variables = lookup(name=product, variables=variables)
variables = variables["dl_name"].values.tolist()
td = timedelta(days=30)
current_start = startdate
while current_start <= enddate:
current_end = current_start + td
if current_end >= enddate:
current_end = enddate
fname = "{start}_{end}.{ext}".format(
start=current_start.strftime("%Y%m%d"),
end=current_end.strftime("%Y%m%d"),
ext="grb" if grb else "nc",
)
downloaded_data_path = os.path.join(target_path, "temp_downloaded")
if not os.path.exists(downloaded_data_path):
os.mkdir(downloaded_data_path)
dl_file = os.path.join(downloaded_data_path, fname)
download_eraint(
dl_file,
current_start,
current_end,
variables,
grid_size=grid_size,
h_steps=h_steps,
type=type,
steps=steps,
grb=grb,
dry_run=dry_run,
)
if grb:
save_gribs_from_grib(dl_file, target_path, product.upper())
else:
save_ncs_from_nc(dl_file, target_path, product.upper())
if not keep_original:
shutil.rmtree(downloaded_data_path)
current_start = current_end + timedelta(days=1)