def ecwmf_surface_pressure(input_path, lonlat, time):
"""
Retrieve a pixel value from the ECWMF Surface Pressure
collection.
Scales the result by 100 before returning.
"""
product = DatasetName.SURFACE_PRESSURE.value.lower()
search = pjoin(input_path, DatasetName.ECMWF_PATH_FMT.value)
files = glob.glob(search.format(product=product, year=time.year))
data = None
required_ymd = datetime.datetime(time.year, time.month, time.day)
for f in files:
url = urlparse(f, scheme='file').geturl()
ymd = splitext(basename(f))[0].split('_')[1]
ancillary_ymd = datetime.datetime.strptime(ymd, '%Y-%m-%d')
if ancillary_ymd == required_ymd:
data = get_pixel(f, lonlat) / 100.0
metadata = {
'data_source': 'ECWMF Surface Pressure',
'url': url,
'query_date': time
}
# ancillary metadata tracking
md = extract_ancillary_metadata(f)
for key in md:
metadata[key] = md[key]
return data, metadata
if data is None:
raise AncillaryError("No ECWMF Surface Pressure data")
def get_elevation_data(lonlat, pathname):
"""
Get elevation data for a scene.
:param lon_lat:
The latitude, longitude of the scene center.
:type lon_lat:
float (2-tuple)
:pathname:
The pathname of the DEM with a ':' to seperate the
dataset name.
:type dem_dir:
str
"""
fname, dname = pathname.split(":")
try:
data, md_uuid = get_pixel(fname, dname, lonlat)
data = data * 0.001 # scale to correct units
except ValueError:
raise AncillaryError("No Elevation data")
metadata = {
"id": numpy.array([md_uuid], VLEN_STRING),
}
return data, metadata
def ecwmf_water_vapour(input_path, lonlat, time):
"""
Retrieve a pixel value from the ECWMF Total Column Water Vapour
collection.
"""
product = DatasetName.WATER_VAPOUR.value.lower()
search = pjoin(input_path, DatasetName.ECMWF_PATH_FMT.value)
files = glob.glob(search.format(product=product, year=time.year))
data = None
required_ymd = datetime.datetime(time.year, time.month, time.day)
for f in files:
url = urlparse(f, scheme='file').geturl()
ymd = splitext(basename(f))[0].split('_')[1]
ancillary_ymd = datetime.datetime.strptime(ymd, '%Y-%m-%d')
if ancillary_ymd == required_ymd:
data = get_pixel(f, lonlat)
metadata = {
'data_source': 'ECWMF Total Column Water Vapour',
'url': url,
'query_date': time
}
# ancillary metadata tracking
md = extract_ancillary_metadata(f)
for key in md:
metadata[key] = md[key]
return data, metadata
if data is None:
raise AncillaryError("No ECWMF Total Column Water Vapour data")
def get_elevation_data(lonlat, dem_path):
"""
Get elevation data for a scene.
:param lon_lat:
The latitude, longitude of the scene center.
:type lon_lat:
float (2-tuple)
:dem_dir:
The directory in which the DEM can be found.
:type dem_dir:
str
"""
datafile = pjoin(dem_path, "DEM_one_deg.tif")
url = urlparse(datafile, scheme='file').geturl()
try:
data = get_pixel(datafile, lonlat) * 0.001 # scale to correct units
except IndexError:
raise AncillaryError("No Elevation data")
metadata = {'data_source': 'Elevation', 'url': url}
# ancillary metadata tracking
md = extract_ancillary_metadata(datafile)
for key in md:
metadata[key] = md[key]
return data, metadata
def get_water_vapour(acquisition, water_vapour_dict, scale_factor=0.1):
"""
Retrieve the water vapour value for an `acquisition` and the
path for the water vapour ancillary data.
"""
dt = acquisition.acquisition_datetime
geobox = acquisition.gridded_geo_box()
year = dt.strftime('%Y')
filename = "pr_wtr.eatm.{year}.tif".format(year=year)
if 'user' in water_vapour_dict:
metadata = {'data_source': 'User defined value'}
return water_vapour_dict['user'], metadata
else:
water_vapour_path = water_vapour_dict['pathname']
datafile = pjoin(water_vapour_path, filename)
url = urlparse(datafile, scheme='file').geturl()
# calculate the water vapour band number based on the datetime
doy = dt.timetuple().tm_yday
hour = dt.timetuple().tm_hour
band = (int(doy) - 1) * 4 + int((hour + 3) / 6)
# Check for boundary condition: 1 Jan, 0-3 hours
if band == 0 and doy == 1:
band = 1
# Get the number of bands
with rasterio.open(datafile) as src:
n_bands = src.count
# Enable NBAR Near Real Time (NRT) processing
if band > (n_bands + 1):
rasterdoy = (((n_bands) - (int((hour + 3) / 6))) / 4) + 1
if (doy - rasterdoy) < 7:
band = (int(rasterdoy) - 1) * 4 + int((hour + 3) / 6)
try:
data = get_pixel(datafile, geobox.centre_lonlat, band=band)
except IndexError:
raise AncillaryError("No Water Vapour data")
data = data * scale_factor
metadata = {'data_source': 'Water Vapour', 'url': url, 'query_date': dt}
# ancillary metadata tracking
md = extract_ancillary_metadata(datafile)
for key in md:
metadata[key] = md[key]
return data, metadata
def get_ozone_data(ozone_fname, lonlat, time):
"""
Get ozone data for a scene. `lonlat` should be the (x,y) for the centre
the scene.
"""
dname = time.strftime("%b").lower()
try:
data, md_uuid = get_pixel(ozone_fname, dname, lonlat)
except ValueError:
raise AncillaryError("No Ozone data")
metadata = {
"id": numpy.array([md_uuid], VLEN_STRING),
"tier": OzoneTier.DEFINITIVE.name,
}
return data, metadata
def ecwmf_geo_potential(input_path, lonlat, time):
"""
Retrieve a pixel value from the ECWMF Geo-Potential collection
across 37 height pressure levels, for a given longitude,
latitude and time.
Converts to geo-potential height in KM, and reverses the order of
the elements (1000 -> 1 mb, rather than 1 -> 1000 mb) before
returning.
"""
product = DatasetName.GEOPOTENTIAL.value.lower()
search = pjoin(input_path, DatasetName.ECMWF_PATH_FMT.value)
files = glob.glob(search.format(product=product, year=time.year))
data = None
required_ymd = datetime.datetime(time.year, time.month, time.day)
for f in files:
url = urlparse(f, scheme='file').geturl()
ymd = splitext(basename(f))[0].split('_')[1]
ancillary_ymd = datetime.datetime.strptime(ymd, '%Y-%m-%d')
if ancillary_ymd == required_ymd:
bands = list(range(1, 38))
data = get_pixel(f, lonlat, bands)[::-1]
scaled_data = data / 9.80665 / 1000.0
metadata = {
'data_source': 'ECWMF Geo-Potential',
'url': url,
'query_date': time
}
# ancillary metadata tracking
md = extract_ancillary_metadata(f)
for key in md:
metadata[key] = md[key]
# internal file metadata (and reverse the ordering)
df = read_metadata_tags(f, bands).iloc[::-1]
df.insert(0, 'GeoPotential', data)
df.insert(1, 'GeoPotential_Height', scaled_data)
return df, md
if data is None:
raise AncillaryError("No ECWMF Geo-Potential profile data")
def ecwmf_relative_humidity(input_path, lonlat, time):
"""
Retrieve a pixel value from the ECWMF Relative Humidity collection
across 37 height pressure levels, for a given longitude,
latitude and time.
Reverses the order of elements
(1000 -> 1 mb, rather than 1 -> 1000 mb) before returning.
"""
product = DatasetName.RELATIVE_HUMIDITY.value.lower()
search = pjoin(input_path, DatasetName.ECMWF_PATH_FMT.value)
files = glob.glob(search.format(product=product, year=time.year))
data = None
required_ymd = datetime.datetime(time.year, time.month, time.day)
for f in files:
url = urlparse(f, scheme='file').geturl()
ymd = splitext(basename(f))[0].split('_')[1]
ancillary_ymd = datetime.datetime.strptime(ymd, '%Y-%m-%d')
if ancillary_ymd == required_ymd:
bands = list(range(1, 38))
data = get_pixel(f, lonlat, bands)[::-1]
metadata = {
'data_source': 'ECWMF Relative Humidity',
'url': url,
'query_date': time
}
# file level metadata
md = extract_ancillary_metadata(f)
for key in md:
metadata[key] = md[key]
# internal file metadata (and reverse the ordering)
df = read_metadata_tags(f, bands).iloc[::-1]
df.insert(0, 'Relative_Humidity', data)
return df, metadata
if data is None:
raise AncillaryError("No ECWMF Relative Humidity profile data")
def ecwmf_elevation(datafile, lonlat):
"""
Retrieve a pixel from the ECWMF invariant geo-potential
dataset.
Converts to Geo-Potential height in KM.
2 metres is added to the result before returning.
"""
try:
data = get_pixel(datafile, lonlat) / 9.80665 / 1000.0 + 0.002
except IndexError:
raise AncillaryError("No Invariant Geo-Potential data")
url = urlparse(datafile, scheme='file').geturl()
metadata = {'data_source': 'ECWMF Invariant Geo-Potential', 'url': url}
# ancillary metadata tracking
md = extract_ancillary_metadata(datafile)
for key in md:
metadata[key] = md[key]
return data, metadata
def get_ozone_data(ozone_path, lonlat, time):
"""
Get ozone data for a scene. `lonlat` should be the (x,y) for the centre
the scene.
"""
filename = time.strftime('%b').lower() + '.tif'
datafile = pjoin(ozone_path, filename)
url = urlparse(datafile, scheme='file').geturl()
try:
data = get_pixel(datafile, lonlat)
except IndexError:
raise AncillaryError("No Ozone data")
metadata = {'data_source': 'Ozone', 'url': url, 'query_date': time}
# ancillary metadata tracking
md = extract_ancillary_metadata(datafile)
for key in md:
metadata[key] = md[key]
return data, metadata
def get_water_vapour(acquisition,
water_vapour_dict,
scale_factor=0.1,
tolerance=1):
"""
Retrieve the water vapour value for an `acquisition` and the
path for the water vapour ancillary data.
"""
dt = acquisition.acquisition_datetime
geobox = acquisition.gridded_geo_box()
year = dt.strftime("%Y")
hour = dt.timetuple().tm_hour
filename = "pr_wtr.eatm.{year}.h5".format(year=year)
if "user" in water_vapour_dict:
metadata = {
"id": numpy.array([], VLEN_STRING),
"tier": WaterVapourTier.USER.name
}
return water_vapour_dict["user"], metadata
water_vapour_path = water_vapour_dict["pathname"]
datafile = pjoin(water_vapour_path, filename)
if os.path.isfile(datafile):
with h5py.File(datafile, "r") as fid:
index = read_h5_table(fid, "INDEX")
# set the tolerance in days to search back in time
max_tolerance = -datetime.timedelta(days=tolerance)
# only look for observations that have occured in the past
time_delta = index.timestamp - dt
result = time_delta[(time_delta < datetime.timedelta())
& (time_delta > max_tolerance)]
if not os.path.isfile(datafile) or result.shape[0] == 0:
if "fallback_dataset" not in water_vapour_dict:
raise AncillaryError("No actual or fallback water vapour data.")
tier = WaterVapourTier.FALLBACK_DATASET
month = dt.strftime("%B-%d").upper()
# closest previous observation
# i.e. observations are at 0000, 0600, 1200, 1800
# and an acquisition hour of 1700 will use the 1200 observation
observations = numpy.array([0, 6, 12, 18])
hr = observations[numpy.argmin(numpy.abs(hour - observations))]
dataset_name = "AVERAGE/{}/{:02d}00".format(month, hr)
datafile = water_vapour_dict["fallback_dataset"]
else:
tier = WaterVapourTier.DEFINITIVE
# get the index of the closest water vapour observation
# which would be the maximum timedelta
# as we're only dealing with negative timedelta's here
idx = result.idxmax()
record = index.iloc[idx]
dataset_name = record.dataset_name
try:
data, md_uuid = get_pixel(datafile, dataset_name, geobox.centre_lonlat)
except ValueError:
# h5py raises a ValueError not an IndexError for out of bounds
raise AncillaryError("No Water Vapour data")
# the metadata from the original file says (Kg/m^2)
# so multiply by 0.1 to get (g/cm^2)
data = data * scale_factor
metadata = {"id": numpy.array([md_uuid], VLEN_STRING), "tier": tier.name}
return data, metadata
