def ECMWFdload(bdate,hr,filedir,model='ERA5',datatype='fc',humidity='Q',snwe=None,flist=None):
'''
ECMWF data downloading.
Args:
* bdate : date to download (str)
* hr : hour to download (str)
* filedir : files directory (str)
* model : weather model ('ERA5', 'ERAINT', 'HRES')
* datatype : reanalysis data type (an)
* snwe : area extent (tuple of int)
* humidity : humidity
'''
#-------------------------------------------
# Initialize
# Check data
assert model in ('ERA5', 'ERAINT', 'HRES'), 'Unknown model for ECMWF: {}'.format(model)
# Infos for downloading
if model in 'ERAINT':
print('WARNING: you are downloading from the old ECMWF platform. '
'ERA-Interim is deprecated, use ERA-5 instead.')
if model in 'ERA5':
print('INFO: You are using the latest ECMWF platform for downloading datasets: '
'https://cds.climate.copernicus.eu/api/v2')
#-------------------------------------------
# Define parameters
# Humidity
assert humidity in ('Q','R'), 'Unknown humidity field for ECMWF'
if humidity in 'Q':
if model in 'ERA5':
humidparam = 'specific_humidity'
else:
humidparam = 133
elif humidity in 'R':
if model in 'ERA5':
humidparam = 'relative_humidity'
else:
humidparam = 157
# Grid size (only for HRES and ERA-Interim)
if model in 'HRES':
gridsize = '0.10/0.10'
elif model in 'ERAINT':
gridsize = '0.75/0.75'
#-------------------------------------------
# file name
if not flist:
flist = []
for k in range(len(bdate)):
day = bdate[k]
if model == 'ERA5':
fname = os.path.join(filedir, 'ERA-5_{}_{}.grb'.format(day, hr))
elif model == 'ERAINT':
fname = os.path.join(filedir, 'ERA-Int_{}_{}.grb'.format(day, hr))
elif model in 'HRES':
fname = os.path.join(filedir, 'HRES_{}_{}.grb'.format(day, hr))
else:
raise ValueError('unrecognized model input: {}'.format(model))
flist.append(fname)
# Iterate over dates
for k in range(len(bdate)):
day = bdate[k]
fname = flist[k]
#-------------------------------------------
# CASE 1: request for CDS API client (new ECMWF platform, for ERA-5)
if model in 'ERA5':
url = 'https://cds.climate.copernicus.eu/api/v2'
key = config.get('CDS', 'key')
# Contact the server
c = cdsapi.Client(url=url, key=key)
# Pressure levels
pressure_lvls = ['1','2','3','5','7','10','20','30','50',
'70','100','125','150','175','200','225',
'250','300','350','400','450','500','550',
'600','650','700','750','775','800','825',
'850','875','900','925','950','975','1000']
# Dictionary
indict = {'product_type' :'reanalysis',
'format' :'grib',
'variable' :['geopotential','temperature','{}'.format(humidparam)],
'pressure_level' : pressure_lvls,
'year' :'{}'.format(day[0:4]),
'month' :'{}'.format(day[4:6]),
'day' :'{}'.format(day[6:8]),
'time' :'{}:00'.format(hr)}
# download a geographical area subset
if snwe is not None:
s, n, w, e = snwe
indict['area'] = '/'.join(['{:.2f}'.format(x) for x in [n, w, s, e]])
# Assert grib file not yet downloaded
if not os.path.exists(fname):
print('Downloading %d of %d: %s '%(k+1,len(bdate), fname))
print(indict)
# Make the request
c.retrieve('reanalysis-{}-pressure-levels'.format(model.lower()),indict,target=fname)
#-------------------------------------------
# CASE 2: request for WEB API client (old ECMWF platform, deprecated, for ERA-Int and HRES)
else:
# Contact the server
from pyaps3.ecmwfapi import ECMWFDataServer
url = "https://api.ecmwf.int/v1"
emid = config.get('ECMWF', 'email')
key = config.get('ECMWF', 'key')
server = ECMWFDataServer(url=url, key=key, email=emid)
# Dictionary
indict = {'dataset' : '{}'.format(model),
'type' : '{}'.format(datatype),
'date' : '{}'.format(day),
'time' : '{}'.format(hr),
'step' : '0',
'levtype' : "pl",
'levelist' : "all",
'grid' : '{}'.format(gridsize),
'param' : '129/130/{}'.format(humidparam),
'target' : '{}'.format(fname)}
# Assert grib file not yet downloaded
if not os.path.exists(fname):
print('Downloading %d of %d: %s '%(k+1,len(bdate),fname))
print(indict)
# Make the request
server.retrieve(indict)
return flist
import json, sys, cdsapi
with open("/input/request.json") as req:
request = json.load(req)
cds = cdsapi.Client(request.get("url"),
request.get("uuid") + ":" + request.get("key"))
cds.retrieve(
request.get("variable"),
request.get("options"),
"/output/" + request.get("filename"),
)
#!/usr/bin/env python
import cdsapi
import os
datadir = '/n/holyscratch01/kuang_lab/nwong/TropicalRCE/data/reanalysis/t_air/'
c = cdsapi.Client()
if not os.path.exists(datadir):
os.makedirs(datadir)
for yr in range(1979, 2020):
for mo in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]:
c.retrieve(
'reanalysis-era5-pressure-levels-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis_by_hour_of_day',
'variable':
'temperature',
'year':
yr,
'pressure_level': [
50, 70, 100, 125, 150, 175, 200, 225, 250, 300, 350, 400,
450, 500, 550, 600, 650, 700, 750, 775, 800, 825, 850, 875,
900, 925, 950, 975, 1000
],
'month':
mo,
'time': [
#!/home/smartmet/anaconda3/envs/xr/bin/python
#/usr/bin/env python
import sys
import cdsapi
if (len(sys.argv) > 2):
year = sys.argv[1]
month = sys.argv[2]
day = sys.argv[3]
c = cdsapi.Client(url="https://ads.atmosphere.copernicus.eu/api/v2",
key="1990:211cb5c3-3923-4f49-a5cd-214a4c3bc2b0")
c.retrieve(
'cams-europe-air-quality-forecasts', {
'variable': 'pm10_wildfires',
'model': 'ensemble',
'level': '0',
'date': '%s-%s-01/%s-%s-%s' % (year, month, year, month, day),
'type': ['analysis', 'forecast'],
'time': [
'00:00',
'12:00',
],
'leadtime_hour': ['0', '12', '24', '36', '48', '60', '72', '84', '96'],
'format': 'grib',
}, '/home/smartmet/data/grib/CAMSE_%s0101T000000_%s%s%s_WFPM10_12h.grib' %
(year, year, month, day))
def retrieve(year, month, variable, ndays):
'''
Use ECMWF API to get the data
4.5GB per month --> 55GB per year, 50mins per month of processing
'''
if variable == "2m_temperature":
varlist = ["2m_temperature", "land_sea_mask"]
elif variable == "total_precipitation":
varlist = ["total_precipitation"]
else:
print("please provide correct variable to download")
return
days = ["{:2d}".format(d + 1) for d in range(ndays)]
c = cdsapi.Client()
c.retrieve(
'reanalysis-era5-single-levels', {
'product_type':
'reanalysis',
'format':
'netcdf',
'variable':
varlist,
'year':
"{}".format(year),
'month':
"{:02d}".format(month),
'day':
days,
'time': [
'00:00',
'01:00',
'02:00',
'03:00',
'04:00',
'05:00',
'06:00',
'07:00',
'08:00',
'09:00',
'10:00',
'11:00',
'12:00',
'13:00',
'14:00',
'15:00',
'16:00',
'17:00',
'18:00',
'19:00',
'20:00',
'21:00',
'22:00',
'23:00',
]
},
os.path.join(utils.DATALOC,
"{}{:02d}_hourly_{}.nc".format(year, month, variable)))
time.sleep(5) # to allow any writing process to finish up.
# make a "success" file
with open(
os.path.join(
utils.DATALOC,
"{}{:02d}_{}_success.txt".format(year, month, variable)),
"w") as outfile:
outfile.write("Success {}".format(dt.datetime.now()))
return # retreive
def main():
import cdsapi
import numpy as np
import os
import pandas as pd
import math
def quarter_up(x):
return math.ceil(x * 4) / 4
def quarter_down(x):
return math.floor(x * 4) / 4
c = cdsapi.Client()
file = '/Volumes/Neely/BioDAR/ERA5/sites of light and suction traps.xlsx'
suction_traps = pd.read_excel(file, header=0, sheet_name='Suction traps')
number_of_traps = len(suction_traps['Lat'])
areas = []
trap_name = []
for a in range(0, number_of_traps):
lats = [quarter_up(suction_traps['Lat'][a]), quarter_down(suction_traps['Lat'][a])]
longs = [quarter_up(suction_traps['Long'][a]), quarter_down(suction_traps['Long'][a])]
areas.append([max(lats), min(longs), min(lats), max(longs), ])
trap_name.append(suction_traps['Trap name'][a].replace(" ", "_"))
start_year = 1979
stop_year = 2021
years = np.arange(start_year, stop_year + 1)
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
days = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17',
'18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31']
for year in years:
for month in months:
for day in days:
for idx, area in enumerate(areas):
try:
outdir = '/Volumes/Neely/BioDAR/ERA5/Myrna_TrapLocations_0_25_Box/suction_traps/surface/' \
+ str(trap_name[idx]) + '/'
if not os.path.exists(outdir):
os.makedirs(outdir)
file_name = outdir + 'era5_surface_' + str(trap_name[idx]) + '_' + \
str(year) + str(month) + str(day) + '.nc'
print(str(trap_name[idx]), area)
print(file_name)
if os.path.isfile(file_name) == True:
print('exists')
continue
else:
c.retrieve('reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'format': 'netcdf',
'variable': ['10m_u_component_of_wind', '10m_v_component_of_wind',
'10m_wind_gust_since_previous_post_processing',
'2m_dewpoint_temperature', '2m_temperature',
'skin_temperature',
'boundary_layer_dissipation',
'boundary_layer_height',
'cloud_base_height',
'evaporation', 'potential_evaporation',
'leaf_area_index_high_vegetation',
'leaf_area_index_low_vegetation',
'low_vegetation_cover', 'high_vegetation_cover',
'type_of_high_vegetation', 'type_of_low_vegetation',
'maximum_2m_temperature_since_previous_post_processing',
'minimum_2m_temperature_since_previous_post_processing',
'mean_sea_level_pressure',
'orography',
'runoff', 'sub_surface_runoff', 'surface_runoff',
'surface_latent_heat_flux',
'surface_net_solar_radiation',
'surface_net_thermal_radiation', 'surface_pressure',
'surface_sensible_heat_flux',
'surface_solar_radiation_downwards',
'surface_thermal_radiation_downwards',
'total_cloud_cover',
'total_column_rain_water',
'precipitation_type', 'total_precipitation',
],
'year': [str(year)],
'month': [month],
'day': [day],
'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00',
'08:00', '09:00', '10:00', '11:00', '12:00', '13:00', '14:00', '15:00',
'16:00', '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00',
],
'area': area,
}, file_name)
except:
continue
def Download_ERA5():
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 2 14:30:08 2020
@author: Eric Johnson
"""
# xarray for netcdfs
# ERA5 Login Credentials:
# username: [email protected]
# password: DataScience2020
# Extent: Longitude (-115.86289, -94.9989) Latitude (-79.9461, -75.58234)
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# This section modified from https://stackoverflow.com/questions/44210656/how-to-check-if-a-module-is-installed-in-python-and-if-not-install-it-within-t/44210735
# Check to insure all necessary packages are installed, install missing packages
# import sys
# import subprocess
# import pkg_resources
# required = {'cdsapi', 'netCDF4'}
# installed = {pkg.key for pkg in pkg_resources.working_set}
# missing = required - installed
# if missing:
# python = sys.executable
# subprocess.check_call([python, '-m', 'pip', 'install', *missing], stdout=subprocess.DEVNULL)
###############################################################################
## Download ERA5 variables into files
# url = 'https://cds.climate.copernicus.eu/api/v2'
# key = '38293:414bf99c-171e-48e6-b13d-75661652a8de'
# login_file = '.cdsapirc'
# 10-m u-component of wind
c = cdsapi.Client()
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'10m_u_component_of_wind',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('10m_u_component_of_wind.nc')
# 10-m v component of wind
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'10m_v_component_of_wind',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('10m_v_component_of_wind.nc')
# Total precipitation
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'total_precipitation',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('total_precipitation.nc')
return
def download_era5(start, end, data_path, domain=None):
c = cdsapi.Client()
if domain is not None:
plot_bounds = domains[domain]
else:
plot_bounds = domains['US']
times = defaultdict(list)
while start <= end:
time_string = datetime.strftime(start, '%Y%m%d_%H')
times['time-str'].append(time_string)
times['years'].append(str(start.year))
times['months'].append(str(start.month).zfill(2))
times['days'].append(str(start.day).zfill(2))
times['hours'].append(str(start.hour).zfill(2) + ':00')
start += timedelta(hours=1)
years = sorted(set(times['years']))
months = sorted(set(times['months']))
days = sorted(set(times['days']))
hours = sorted(set(times['hours']))
outfile = '%s-%s.nc' % (times['time-str'][0], times['time-str'][-1])
'''
# Surface Level variables (need for Pressure and Geopotential calculations)
c.retrieve(
'reanalysis-era5-complete',
{
'class': 'ea',
'date': '2020-04-01/to/2020-04-30',
'expver': '1',
'levelist': '1',
'levtype': 'ml',
'param': '129/152',
'stream': 'oper',
'time': '00:00:00/01:00:00/02:00:00/03:00:00/04:00:00/05:00:00/06:00:00/07:00:00/08:00:00/09:00:00/10:00:00/11:00:00/12:00:00/13:00:00/14:00:00/15:00:00/16:00:00/17:00:00/18:00:00/19:00:00/20:00:00/21:00:00/22:00:00/23:00:00',
'type': 'an',
'area' : '%s/%s/%s/%s' % (plot_bounds[-1]+5,plot_bounds[0]-8,plot_bounds[1]-3,plot_bounds[2]+3),
'grid': '0.25/0.25',
'format': 'netcdf'
},
'%s/%s' % (data_path, outfile))
'''
# Model Level variables
c.retrieve(
'reanalysis-era5-complete', {
'class':
'ea',
'date':
'2020-05-01/to/2020-05-31',
'expver':
'1',
'levelist':
'1/57/58/59/60/61/62/63/64/65/66/67/68/69/70/71/72/73/74/75/76/77/78/79/80/81/82/83/84/85/86/87/88/89/90/91/92/93/94/95/96/97/98/99/100/101/102/103/104/105/106/107/108/109/110/111/112/113/114/115/116/117/118/119/120/121/122/123/124/125/126/127/128/129/130/131/132/133/134/135/136/137',
'levtype':
'ml',
'param':
'129/130/131/132/133/152',
'stream':
'oper',
'time':
'00:00:00/01:00:00/02:00:00/03:00:00/04:00:00/05:00:00/06:00:00/07:00:00/08:00:00/09:00:00/10:00:00/11:00:00/12:00:00/13:00:00/14:00:00/15:00:00/16:00:00/17:00:00/18:00:00/19:00:00/20:00:00/21:00:00/22:00:00/23:00:00',
'type':
'an',
'area':
'%s/%s/%s/%s' % (plot_bounds[-1] + 5, plot_bounds[0] - 8,
plot_bounds[1], plot_bounds[2]),
'grid':
'0.25/0.25',
'format':
'netcdf'
}, '%s/%s' % (data_path, outfile))
"""
if not os.path.exists(data_path + '/' + outfile):
c.retrieve(
'reanalysis-era5-pressure-levels',
{
'product_type': 'reanalysis',
'variable': DEFAULT_DATA[pressure_vars],
'pressure_level': DEFAULT_DATA[pressure_levs],
'year': years,
'month': months,
'day': days,
'time': hours,
'format': 'netcdf',
'area' : [plot_bounds[-1]+5,plot_bounds[0]-8,plot_bounds[1],plot_bounds[2]],
'grid': [0.25, 0.25],
},
'%s/%s' % (data_path, outfile))
# Surface and other single-level variables
if not os.path.exists(data_path + '/sfc_' + outfile):
c.retrieve(
'reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'variable': DEFAULT_DATA[surface_vars],
'year': times['years'],
'month': times['months'],
'day': times['days'],
'time': times['hours'],
'format': 'netcdf',
'area' : [plot_bounds[-1]+1,plot_bounds[0]-1,plot_bounds[1]-1,plot_bounds[2]+1],
'grid': [0.25, 0.25],
},
'%s/sfc_%s' % (data_path, outfile))
"""
return
def download_ERA5_hourly(varnames,
years,
months,
days,
output_dir,
filename_suffix=None):
'''
This script downloads ERA5 hourly data, with one file with 24 hours of data
for each day.
Parameters
----------
varnames : list of strings
List of ERA5 variables. Names can be found under "Show API request":
https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-land?tab=form
years : list of strings
List with each year of interest from 1981-present in 'YYYY' format.
months : list of strings
List with each month of interest in 'MM' format..
days : list of strings
List with each day of interest in 'DD' format..
output_dir : str
filepath for output files
filename_suffix: str (optional)
optional string to add to the end of filename (e.g. describing variables)
Returns
-------
ERA5 hourly data, with one file for each day, saved to the output directory.
'''
os.chdir(output_dir)
c = cdsapi.Client()
# File naming
if filename_suffix == None:
fn_suffix = ''
else:
fn_suffix = '_' + filename_suffix
for year in years:
for month in months:
for day in days:
c.retrieve(
'reanalysis-era5-land', {
'format':
'netcdf',
'time': [
'00:00',
'01:00',
'02:00',
'03:00',
'04:00',
'05:00',
'06:00',
'07:00',
'08:00',
'09:00',
'10:00',
'11:00',
'12:00',
'13:00',
'14:00',
'15:00',
'16:00',
'17:00',
'18:00',
'19:00',
'20:00',
'21:00',
'22:00',
'23:00',
],
'day':
day,
'month':
month,
'year':
year,
'variable':
varnames
},
'{}-{}-{}_ERA5_hourly{}.nc'.format(year, month, day,
fn_suffix))
https://software.ecmwf.int/wiki/display/CKB/ERA5+data+documentation
.. _pre-filled form:
http://apps.ecmwf.int/data-catalogues/era5/?stream=oper&levtype=ml&expver=1&month=jan&year=2008&type=an&class=ea
"""
import cdsapi
import re
import os
import datetime as dt
from multiprocessing import Process
from config import area, era5_data_dir, model_level_file_name_format, surface_file_name_format, \
start_year, final_year, upper_level
client = cdsapi.Client() # Connect to server.
def area_type(s):
"""Validate the type of area argument.
Args:
s (str): The parsed argument.
Returns:
str: The input string if valid.
Raises:
ValueError: If `s` does not match the regex.
"""
def download_hourly(self, start, end, destination):
"""Downloads hourly files for given time range and stores at specified location.
Hourly data products are saved per hour and monthly data products are
saved per month. Note that you have to install the CDS API key before
download is possible: https://cds.climate.copernicus.eu/api-how-to
Args:
start(datetime.datetime): start date and time (year, month, day,
hour), if hour is not specified for hourly dataproduct, all
hours are downloaded for each date.
end(datetime.datetime): end date and time (year, month, day, hour),
if hour is not specified for hourly dataproduct, all hours are
downloaded for each date.
destination(``str`` or ``pathlib.Path``): path to directory where
the downloaded files should be stored.
"""
# open new client instance
with _create_cds_api_rc():
c = cdsapi.Client()
# subset region, if requested
if self.product.domain == "global":
area = ""
domain = ""
else:
# change to requested order of coordinate values
dom = np.array([
self.product.domain[1],
self.product.domain[2],
self.product.domain[0],
self.product.domain[3],
]).astype(str)
area = "/".join(dom)
domain = "-".join(np.array(self.product.domain).astype(str))
dates = self.get_timesteps_hourly(start, end)
# container to save list of downloaded files
files = []
# send API request for each specific hour in time range
for idx, date in enumerate(dates):
# define download parameters for hourly download
year = str(dates[idx].year)
month = str(dates[idx].month)
day = str(dates[idx].day)
hour = str(dates[idx].hour)
# get download key
download_key = "reanalysis"
if "land" in self.product.name:
download_key = ""
# zero padding for day
if int(day) < 10:
day = "0" + str(day)
# zero padding for hour string in filename
hourstr = str(hour)
if int(hour) < 10:
hourstr = "0" + str(hour)
# zero padding for month
if int(month) < 10:
month = "0" + str(month)
filename = (self.product.name + "_" + year + month + day +
hourstr + "_" + "-".join(self.product.variables) +
domain + ".nc")
# set output path and file name
out = Path(str(destination)) / str(filename)
# only download if file not already already exists
if os.path.exists(out):
print(out, " already exists.")
else:
if "pressure" in self.product.name:
c.retrieve(
self.product.name,
{
"product_type":
download_key,
"format":
"netcdf",
"area":
area,
"variable":
self.product.variables,
"pressure_level": [
"1",
"2",
"3",
"5",
"7",
"10",
"20",
"30",
"50",
"70",
"100",
"125",
"150",
"175",
"200",
"225",
"250",
"300",
"350",
"400",
"450",
"500",
"550",
"600",
"650",
"700",
"750",
"775",
"800",
"825",
"850",
"875",
"900",
"925",
"950",
"975",
"1000",
],
"year":
year,
"month":
month,
"day":
day,
"time":
hour,
},
out,
)
else:
c.retrieve(
self.product.name,
{
"product_type": download_key,
"format": "netcdf",
"area": area,
"variable": self.product.variables,
"year": year,
"month": month,
"day": day,
"time": hour,
},
out,
)
print("file downloaded and saved as", out)
files.append(out)
return files
def download_and_move(target_path,
startdate,
enddate,
variables=None,
keep_original=False,
h_steps=[0, 6, 12, 18],
grb=False,
dry_run=False):
"""
Downloads the data from the ECMWF servers and moves them to the target path.
This is done in 30 day increments between start and end date.
The files are then extracted into separate grib files per parameter and stored
in yearly folders under the target_path.
Parameters
----------
target_path : str
Path where the files are stored to
startdate: datetime
first date to download
enddate: datetime
last date to download
variables : list, optional (default: None)
Name of variables to download
keep_original: bool
keep the original downloaded data
h_steps: list
List of full hours to download data at the selected dates e.g [0, 12]
grb: bool, optional (default: False)
Download data as grib files
dry_run: bool
Do not download anything, this is just used for testing the functions
"""
if variables is None:
variables = default_variables()
else:
# find the dl_names
variables = lookup(name='ERA5', variables=variables)
variables = variables['dl_name'].values.tolist()
curr_start = startdate
if dry_run:
warnings.warn('Dry run does not create connection to CDS')
c = None
else:
c = cdsapi.Client()
while curr_start <= enddate:
sy, sm, sd = curr_start.year, curr_start.month, curr_start.day
sm_days = calendar.monthrange(sy, sm)[1] # days in the current month
y, m = sy, sm
if (enddate.year == y) and (enddate.month == m):
d = enddate.day
else:
d = sm_days
curr_end = datetime(y, m, d)
fname = '{start}_{end}.{ext}'.format(
start=curr_start.strftime("%Y%m%d"),
end=curr_end.strftime("%Y%m%d"),
ext='grb' if grb else 'nc')
downloaded_data_path = os.path.join(target_path, 'temp_downloaded')
if not os.path.exists(downloaded_data_path):
os.mkdir(downloaded_data_path)
dl_file = os.path.join(downloaded_data_path, fname)
finished, i = False, 0
while (not finished) and (i < 5): # try max 5 times
try:
finished = download_era5(c,
years=[sy],
months=[sm],
days=range(sd, d + 1),
h_steps=h_steps,
variables=variables,
grb=grb,
target=dl_file,
dry_run=dry_run)
break
except:
# delete the partly downloaded data and retry
os.remove(dl_file)
finished = False
i += 1
continue
if grb:
save_gribs_from_grib(dl_file, target_path, product_name='ERA5')
else:
save_ncs_from_nc(dl_file, target_path, product_name='ERA5')
if not keep_original:
shutil.rmtree(downloaded_data_path)
curr_start = curr_end + timedelta(days=1)
def update_cds_monthly_data(
dataset_name="reanalysis-era5-single-levels-monthly-means",
product_type="monthly_averaged_reanalysis",
variables=[
"2m_dewpoint_temperature",
"angle_of_sub_gridscale_orography",
"orography",
"slope_of_sub_gridscale_orography",
"total_column_water_vapour",
"total_precipitation",
],
area=[40, 70, 30, 85],
pressure_level=None,
path="Data/",
qualifier=None,
):
"""
Imports the most recent version of the given monthly ERA5 dataset as a netcdf from the CDS API.
Inputs:
dataset_name: str
prduct_type: str
variables: list of strings
pressure_level: str or None
area: list of scalars
path: str
qualifier: str
Returns: local filepath to netcdf.
"""
if area is str:
qualifier = area
area = basin_extent(area)
now = datetime.datetime.now()
if qualifier == None:
filename = (dataset_name + "_" + product_type + "_" +
now.strftime("%m-%Y") + ".nc")
else:
filename = (dataset_name + "_" + product_type + "_" + qualifier + "_" +
now.strftime("%m-%Y") + ".nc")
filepath = path + filename
# Only download if updated file is not present locally
if not os.path.exists(filepath):
current_year = now.strftime("%Y")
years = np.arange(1979, int(current_year) + 1, 1).astype(str)
months = np.arange(1, 13, 1).astype(str)
c = cdsapi.Client()
if pressure_level == None:
c.retrieve(
"reanalysis-era5-single-levels-monthly-means",
{
"format": "netcdf",
"product_type": product_type,
"variable": variables,
"year": years.tolist(),
"time": "00:00",
"month": months.tolist(),
"area": area,
},
filepath,
)
else:
c.retrieve(
"reanalysis-era5-single-levels-monthly-means",
{
"format": "netcdf",
"product_type": product_type,
"variable": variables,
"pressure_level": pressure_level,
"year": years.tolist(),
"time": "00:00",
"month": months.tolist(),
"area": area,
},
filepath,
)
return filepath
def update_cds_hourly_data(
dataset_name="reanalysis-era5-pressure-levels",
product_type="reanalysis",
variables=["geopotential"],
pressure_level="200",
area=[90, -180, -90, 180],
path="Data/",
qualifier=None,
):
"""
Imports the most recent version of the given hourly ERA5 dataset as a netcdf from the CDS API.
Inputs:
dataset_name: str
prduct_type: str
variables: list of strings
area: list of scalars
pressure_level: str or None
path: str
qualifier: str
Returns: local filepath to netcdf.
"""
now = datetime.datetime.now()
if qualifier == None:
filename = (dataset_name + "_" + product_type + "_" +
now.strftime("%m-%Y") + ".nc")
else:
filename = (dataset_name + "_" + product_type + "_" +
now.strftime("%m-%Y") + "_" + qualifier + ".nc")
filepath = path + filename
# Only download if updated file is not present locally
if not os.path.exists(filepath):
current_year = now.strftime("%Y")
years = np.arange(1979, int(current_year) + 1, 1).astype(str)
months = np.arange(1, 13, 1).astype(str)
days = np.arange(1, 32, 1).astype(str)
c = cdsapi.Client()
if pressure_level == None:
c.retrieve(
dataset_name,
{
"format": "netcdf",
"product_type": product_type,
"variable": variables,
"year": years.tolist(),
"time": "00:00",
"month": months.tolist(),
"day": days.tolist(),
"area": area,
},
filepath,
)
else:
c.retrieve(
dataset_name,
{
"format": "netcdf",
"product_type": product_type,
"variable": variables,
"pressure_level": pressure_level,
"year": years.tolist(),
"time": "00:00",
"month": months.tolist(),
"day": days.tolist(),
"area": area,
},
filepath,
)
return filepath
def grab_era5(month, year, output_folder, region, mylat, mylon):
"""A function to download ERA5 Lande data for one month. Assumes
the Copernicus Data Service API works and is properly configured.
Downloaded files have names `{region}.{year}_{month}.nc`.
Function checks whether the file already exists before requesting
the data, as requesting the data takes a while.
Parameters
----------
month: int
The month number
year: int
The year number
output_folder : str
The output folder where the files will be saved to.
region: str
Name of the region. Useful for saving the data.
mylat : 2-tuple, 2-list
North and South Latitudes in decimal degrees.
mylon : float
West and East Longitudes in decimal degrees.
"""
output_folder = Path(output_folder) / "netcdf"
output_folder.mkdir(parents=True, exist_ok=True)
output_nc_file = (output_folder /
f"ERA5_{region:s}.{year:d}_{month:02d}.nc")
# This is needed to keep getting the updated ERA5 datasets
today = dt.datetime.now()
delta_t = today - dt.datetime(year, month, 1)
if not output_nc_file.exists() or (0 <= delta_t.days <= 120):
LOG.info(f"Downloading {year}-{month}")
# '80/-50/-25/0', # North, West, South, East.
area = (f"{int(mylat[1]):d}/{int(mylon[0]):d}/" +
f"{int(mylat[0]):d}/{int(mylon[1]):d}")
c = cdsapi.Client()
c.retrieve(
"reanalysis-era5-single-levels",
{
"format":
"netcdf",
"variable": [
"10m_u_component_of_wind",
"10m_v_component_of_wind",
"2m_dewpoint_temperature",
"2m_temperature",
"evaporation",
"potential_evaporation",
"surface_solar_radiation_downwards",
"total_precipitation",
"volumetric_soil_water_layer_1",
"volumetric_soil_water_layer_2",
"volumetric_soil_water_layer_3",
"volumetric_soil_water_layer_4",
],
"product_type":
"reanalysis",
"year":
f"{year:d}",
"month":
f"{month:02d}",
"day": [f"{day:02d}" for day in range(1, 32)],
"time": [f"{hour:02d}:00" for hour in range(0, 24)],
"area":
area,
"format":
"netcdf",
},
output_nc_file.as_posix(),
)
return output_nc_file.as_posix()
else:
LOG.info(f"Skipping {year}-{month}")
return None
def get_era5_parms(filename, year):
# Create list of times
hours = 1 # hourly
months = []
for month in range(12):
months.append("{:02d}".format(month + 1))
times = []
for i in range(0, 24, hours):
times.append('{:02d}:00'.format(i))
# Create dictionary defining the request
request = {
'product_type':
'reanalysis',
'format':
'netcdf',
'variable': [
'10m_u_component_of_wind', '10m_v_component_of_wind',
'2m_temperature', '2m_dewpoint_temperature', 'total_precipitation',
'surface_pressure', 'total_cloud_cover',
'surface_solar_radiation_downwards',
'surface_thermal_radiation_downwards',
'surface_solar_radiation_downward_clear_sky'
],
'year':
year,
'month':
months,
'day': [
'01',
'02',
'03',
'04',
'05',
'06',
'07',
'08',
'09',
'10',
'11',
'12',
'13',
'14',
'15',
'16',
'17',
'18',
'19',
'20',
'21',
'22',
'23',
'24',
'25',
'26',
'27',
'28',
'29',
'30',
'31',
],
'time':
times,
'area': [
72,
-10.5,
36.75,
25.5,
],
# 'area': [ 61, -2.0, 50.0, 8.0, ],
# 'grid': ['0.25', '0.25'],
'grid': ['0.75', '0.75'],
}
print('Downloading wind to {}'.format(filename))
print(request)
# Download ERA5 weather
c = cdsapi.Client()
c.retrieve('reanalysis-era5-single-levels', request, filename)
def ECMWFdload(bdate, hr, filedir, model, datatype, humidity='Q'):
'''
ECMWF data downloading.
Args:
* bdate : date to download (str)
* hr : hour to download (str)
* filedir : files directory (str)
* model : weather model ('interim' or 'era5' or 'hres')
* datatype : reanalysis data type (an)
* humidity : humidity
'''
#-------------------------------------------
# Initialize
# Check data
assert model in ('era5', 'interim', 'hres'), 'Unknown model for ECMWF'
# Infos for downloading
if model in 'interim':
print(
'WARNING: you are downloading from the old ECMWF platform. ERA-Interim is deprecated, use ERA-5 instead.'
)
if model in 'era5':
print(
'INFO: You are using the latest ECMWF platform for downloading datasets: https://cds.climate.copernicus.eu/api/v2'
)
#-------------------------------------------
# Define parameters
# Humidity
assert humidity in ('Q', 'R'), 'Unknown humidity field for ECMWF'
if humidity in 'Q':
if model in 'era5':
humidparam = 'specific_humidity'
else:
humidparam = 133
elif humidity in 'R':
if model in 'era5':
humidparam = 'relative_humidity'
else:
humidparam = 157
# Grid size (only for HRES and ERA-Interim)
if model in 'hres':
gridsize = '0.10/0.10'
elif model in 'interim':
gridsize = '0.75/0.75'
#-------------------------------------------
# Iterate over dates
flist = []
for k in range(len(bdate)):
day = bdate[k]
#-------------------------------------------
# CASE 1: request for CDS API client (new ECMWF platform, for ERA-5)
if model in 'era5':
# Contact the server
c = cdsapi.Client()
# Pressure levels
pressure_lvls = [
'1', '2', '3', '5', '7', '10', '20', '30', '50', '70', '100',
'125', '150', '175', '200', '225', '250', '300', '350', '400',
'450', '500', '550', '600', '650', '700', '750', '775', '800',
'825', '850', '875', '900', '925', '950', '975', '1000'
]
# Dictionary
indict = {
'product_type':
'reanalysis',
'format':
'grib',
'variable':
['geopotential', 'temperature', '{}'.format(humidparam)],
'pressure_level':
pressure_lvls,
'year':
'{}'.format(day[0:4]),
'month':
'{}'.format(day[4:6]),
'day':
'{}'.format(day[6:8]),
'time':
'{}:00'.format(hr)
}
# Output
fname = '%s/ERA-5_%s_%s_%s.grb' % (filedir, day, hr, datatype)
flist.append(fname)
# Assert grib file not yet downloaded
if not os.path.exists(fname):
print('Downloading %d of %d: %s ' % (k + 1, len(bdate), fname))
print(indict)
# Make the request
c.retrieve('reanalysis-{}-pressure-levels'.format(model),
indict, fname)
#-------------------------------------------
# CASE 2: request for WEB API client (old ECMWF platform, deprecated, for ERA-Int and HRES)
else:
# Contact the server
from pyaps3.ecmwfapi import ECMWFDataServer
dpath = os.path.dirname(__file__)
config = ConfigParser.RawConfigParser()
config.read(os.path.expanduser('~/.ecmwfcfg'))
url = "https://api.ecmwf.int/v1"
emid = config.get('ECMWF', 'email')
key = config.get('ECMWF', 'key')
server = ECMWFDataServer(url=url, key=key, email=emid)
# Output
if model in 'interim':
fname = '%s/ERA-Int_%s_%s_%s.grb' % (filedir, day, hr,
datatype)
elif model in 'hres':
fname = '%s/HRES_%s_%s_%s.grb' % (filedir, day, hr, datatype)
flist.append(fname)
# Dictionary
indict = {
'dataset': '{}'.format(model),
'type': '{}'.format(datatype),
'date': '{}'.format(day),
'time': '{}'.format(hr),
'step': '0',
'levtype': "pl",
'levelist': "all",
'grid': '{}'.format(gridsize),
'param': '129/130/{}'.format(humidparam),
'target': '{}'.format(fname)
}
# Assert grib file not yet downloaded
if not os.path.exists(fname):
print('Downloading %d of %d: %s ' % (k + 1, len(bdate), fname))
print(indict)
# Make the request
server.retrieve(indict)
return flist
def era5_tcwv_old(imgcol, roi=None):
# prepare imgcol
imgcol = imgcol.sort("system:time_start")
unix_time = imgcol.reduceColumns(
ee.Reducer.toList(), ["system:time_start"]).get('list').getInfo()
def hour_rounder(t):
return (t.replace(second=0, microsecond=0, minute=0, hour=t.hour) +
timedelta(hours=t.minute // 30))
time = [hour_rounder(datetime.fromtimestamp(x / 1000)) for x in unix_time]
dates = [x.strftime('%Y-%m-%d') for x in time]
hour = [time[0].strftime('%H:%M')]
x, y = np.unique(np.array(dates), return_inverse=True)
c = cdsapi.Client()
c.retrieve(
'reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'format': 'grib',
'variable': 'total_column_water_vapour',
"date": dates,
'time': hour,
'area': roi, # N W S E
},
'era5_tcwv.grib')
# get wv raster imfo
wv = gdal.Open('era5_tcwv.grib')
gt = wv.GetGeoTransform()
gt = [round(x, 4) for x in gt]
# client side arrays
wv_array = wv.ReadAsArray()
wv_array = np.round(wv_array * 0.1, 5) # scale from kg/m2 to
wv_array = [wv_array[i] for i in y] # needed because of unique dates
# create list of server-side images
wv_imgs = [ee.Image(ee.Array(x.flatten().tolist(), ee.PixelType.float())) \
#.arrayGet([0]) \
#.arrayProject([0]) \
.arrayFlatten([['WV_SCALED']]) \
.setDefaultProjection(crs='EPSG:4326', crsTransform=gt) for x in wv_array]
wv_imgs = [
wv_imgs[i].rename('WV_SCALED').set('system:time_start', unix_time[i])
for i in range(len(unix_time))
]
wv_img_list = ee.List(wv_imgs)
imgcol_wv = ee.ImageCollection(wv_img_list)
filterTimeEq = ee.Filter.equals(leftField='system:time_start',
rightField='system:time_start')
join_era5 = ee.Join.saveFirst(matchKey='WV_SCALED',
ordering='system:time_start')
imgcol = ee.ImageCollection(
join_era5.apply(imgcol, imgcol_wv, filterTimeEq))
def wv_addband(img):
#wv_scaled = ee.Image(img.get('WV_SCALED')).multiply(0.1).rename('WV_SCALED')
#wv_scaled = wv_scaled.resample('bilinear')
return img.addBands(ee.Image(img.get('WV_SCALED')))
imgcol = imgcol.map(wv_addband)
def add_wv_img(imgcol, wv_img_list):
wv_img_list = ee.List(wv_img_list)
list = imgcol.toList(imgcol.size()).zip(wv_img_list)
list = list.map(lambda x: ee.Image(ee.List(x).get(0)) \
.addBands(ee.Image(ee.List(x).get(1)).rename('WV_SCALED')))
return ee.ImageCollection(list)
#imgcol = add_wv_img(imgcol, wv_imgs)
return imgcol
def callrequest(i, j, c):
print("request received for : ", i, j, c)
duration = 0.001
present_time = time.time()
while time.time() < present_time + duration:
d = c - 1
c = c + 1
if d == 0:
c = cdsapi.Client()
c.retrieve(
'reanalysis-era5-pressure-levels', {
'product_type':
'reanalysis',
'variable':
variables[j],
'pressure_level': [
'50',
'100',
'150',
'200',
'250',
'300',
'350',
'400',
'450',
'500',
'550',
'600',
'650',
'700',
'750',
'800',
'850',
'900',
'950',
'1000',
],
'year':
years[i],
'month': [
'01',
'02',
'03',
'04',
'05',
'06',
'07',
'08',
'09',
'10',
'11',
'12',
],
'day': [
'01',
'02',
'03',
'04',
'05',
'06',
'07',
'08',
'09',
'10',
'11',
'12',
'13',
'14',
'15',
'16',
'17',
'18',
'19',
'20',
'21',
'22',
'23',
'24',
'25',
'26',
'27',
'28',
'29',
'30',
'31',
],
'time': [
'00:00',
'06:00',
'12:00',
'18:00',
],
'area': [
40,
50,
-10,
120,
],
'format':
'netcdf',
}, 'download.nc')
else:
time.sleep(2)
def era5_tcwv(imgcol, roi=None):
# prepare imgcol
imgcol = imgcol.sort("system:time_start")
unix_time = imgcol.reduceColumns(
ee.Reducer.toList(), ["system:time_start"]).get('list').getInfo()
def hour_rounder(t):
return (t.replace(second=0, microsecond=0, minute=0, hour=t.hour) +
timedelta(hours=t.minute // 30))
time = [hour_rounder(datetime.fromtimestamp(x / 1000)) for x in unix_time]
dates = [x.strftime('%Y-%m-%d') for x in time]
hour = [time[0].strftime('%H:%M')]
x, y = np.unique(np.array(dates), return_inverse=True)
c = cdsapi.Client()
c.retrieve(
'reanalysis-era5-single-levels',
{
'product_type': 'reanalysis',
'format': 'grib',
'variable': 'total_column_water_vapour',
"date": dates,
'time': hour,
'area': roi, # N W S E
},
'era5_tcwv.grib')
# get wv raster imfo
wv = gdal.Open('era5_tcwv.grib')
# client side arrays
wv_array = wv.ReadAsArray()
wv_array = wv_array * 0.1 # scale from kg/m2 to
wv_array = [round(np.mean(wv_array[i]), 5)
for i in y] # needed because of unique dates
# create list of server-side images
wv_imgs = [ee.Image.constant(x) for x in wv_array]
wv_imgs = [
wv_imgs[i].rename('WV_SCALED').set('system:time_start', unix_time[i])
for i in range(len(unix_time))
]
wv_img_list = ee.List(wv_imgs)
imgcol_wv = ee.ImageCollection(wv_img_list)
filterTimeEq = ee.Filter.equals(leftField='system:time_start',
rightField='system:time_start')
join_era5 = ee.Join.saveFirst(matchKey='WV_SCALED',
ordering='system:time_start')
imgcol = ee.ImageCollection(
join_era5.apply(imgcol, imgcol_wv, filterTimeEq))
def wv_addband(img):
return img.addBands(ee.Image(img.get('WV_SCALED')))
imgcol = imgcol.map(wv_addband)
return imgcol
def Extract_ERA5():
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 2 14:30:08 2020
@author: Eric Johnson
"""
# xarray for netcdfs
# ERA5 Login Credentials:
# username: [email protected]
# password: DataScience2020
# Extent: Longitude (-115.86289, -94.9989) Latitude (-79.9461, -75.58234)
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# This section modified from https://stackoverflow.com/questions/44210656/how-to-check-if-a-module-is-installed-in-python-and-if-not-install-it-within-t/44210735
# Check to insure all necessary packages are installed, install missing packages
# import sys
# import subprocess
# import pkg_resources
# required = {'cdsapi', 'netCDF4'}
# installed = {pkg.key for pkg in pkg_resources.working_set}
# missing = required - installed
# if missing:
# python = sys.executable
# subprocess.check_call([python, '-m', 'pip', 'install', *missing], stdout=subprocess.DEVNULL)
###############################################################################
## Initial conditions
# Extents
iLat_min = -75.58234
iLat_max = -79.9461
iLon_min = -115.86289
iLon_max = -75.58234
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
## Download ERA5 variables into files
# url = 'https://cds.climate.copernicus.eu/api/v2'
# key = '38293:414bf99c-171e-48e6-b13d-75661652a8de'
# login_file = '.cdsapirc'
# 10-m u-component of wind
c = cdsapi.Client()
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'10m_u_component_of_wind',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('10m_u_component_of_wind.nc')
# 10-m v component of wind
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'10m_v_component_of_wind',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('10m_v_component_of_wind.nc')
# Total precipitation
r = c.retrieve(
'reanalysis-era5-land-monthly-means', {
'format':
'netcdf',
'product_type':
'monthly_averaged_reanalysis',
'variable':
'total_precipitation',
'stream':
'moda',
'year': [
'1980', '1981', '1982', '1983', '1984', '1985', '1986', '1987',
'1988', '1989', '1990', '1991', '1992', '1993', '1994', '1995',
'1996', '1997', '1998', '1999', '2000', '2001', '2002', '2003',
'2004', '2005', '2006', '2007', '2008', '2009', '2010', '2011',
'2012', '2013', '2014', '2015'
],
'month': [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10',
'11', '12'
],
'time':
'00:00'
})
r.download('total_precipitation.nc')
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
## Extract necessary portions of files
# Extract lat/lon data
data = netCDF4.Dataset('10m_u_component_of_wind.nc')
vLat = data.variables['latitude']
vLat = np.asarray(vLat[:])
vLon = data.variables['longitude']
vLon = np.asarray(vLon[:])
vLon[vLon > 180] = -(360 - vLon[vLon > 180]
) # Is this right? What's with the 360 deg latitudes?
# Find indices for required extents
iLat_min_idx = np.argmin(np.abs(vLat - iLat_min))
iLat_max_idx = np.argmin(np.abs(vLat - iLat_max))
iLon_min_idx = np.argmin(np.abs(vLon - iLon_min))
iLon_max_idx = np.argmin(np.abs(vLon - iLon_max))
# Read NetCDF files
data = netCDF4.Dataset('10m_u_component_of_wind.nc')
wind_u_10m = data.variables['u10']
data = netCDF4.Dataset('10m_v_component_of_wind.nc')
wind_v_10m = data.variables['v10']
data = netCDF4.Dataset('total_precipitation.nc')
precip = data.variables['prcp']
# Extract required portions of each variable
wind_u_10m = wind_u_10m[:, iLat_min_idx:iLat_max_idx,
iLon_min_idx:iLon_max_idx]
wind_v_10m = wind_v_10m[:, iLat_min_idx:iLat_max_idx,
iLon_min_idx:iLon_max_idx]
precip = precip[:, iLat_min_idx:iLat_max_idx, iLon_min_idx:iLon_max_idx]
return wind_u_10m, wind_v_10m, precip
f2d = dataset.upper() + '_' + dstr + '.2d'
ftppt = dataset.upper() + '_' + dstr + '.2df'
else:
f3d = options.dir + options.fname + '.3d'
f2d = options.dir + options.fname + '.2d'
ftppt = options.dir + options.fname + '.2df'
if levtype == 'ml':
f3d += '.ml'
file3d = options.dir + f3d
file2d = options.dir + f2d
filetppt = options.dir + ftppt
#server = ECMWFDataServer(verbose=False)
server = cdsapi.Client()
##wtype = "4v" ##4D variational analysis is available as well as analysis.
#wtype="an"
#wtype="reanalysis"
if stream == 'oper':
##need to break each day into four time periods to keep 3d grib files at around 1.6 GB
#wtime1 = "00:00/01:00/02:00/03:00/04:00/05:00"
#wtime2 = "06:00/07:00/08:00/09:00/10:00/11:00"
#wtime3 = "12:00/13:00/14:00/15:00/16:00/17:00"
#wtime4 = "18:00/19:00/20:00/21:00/22:00/23:00"
wtime1 = "00:00/01:00/02:00"
wtime2 = "03:00/04:00/05:00"
def main():
import cdsapi
import numpy as np
import os
c = cdsapi.Client()
outdir = '/Volumes/Neely/GREENLAND/ICECAPSarchive/ERA_5/73N_39_5W_72N_35_5W/raw_data/surface/'
os.chdir(outdir)
start_year = 1979
stop_year = 2020
years = np.arange(start_year, stop_year + 1)
months = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12']
days = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17',
'18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31']
for year in years:
for month in months:
for day in days:
file_name = outdir + 'era5_surface_Summit_' + str(year) + str(month) + str(day) + '.nc'
print(file_name)
if os.path.isfile(file_name)==True:
print('exists')
continue
else:
try:
c.retrieve('reanalysis-era5-single-levels',
{'product_type': 'reanalysis',
'format': 'netcdf',
'variable': ['100m_u_component_of_wind', '100m_v_component_of_wind',
'10m_u_component_of_wind', '10m_v_component_of_wind',
'10m_wind_gust_since_previous_post_processing',
'instantaneous_10m_wind_gust',
'2m_dewpoint_temperature', '2m_temperature',
'maximum_2m_temperature_since_previous_post_processing',
'minimum_2m_temperature_since_previous_post_processing',
'skin_temperature',
'boundary_layer_dissipation', 'boundary_layer_height',
'mean_boundary_layer_dissipation', 'trapping_layer_top_height',
'clear_sky_direct_solar_radiation_at_surface',
'cloud_base_height', 'total_cloud_cover',
'high_cloud_cover', 'medium_cloud_cover', 'low_cloud_cover',
'convective_snowfall', 'convective_snowfall_rate_water_equivalent',
'large_scale_snowfall', 'large_scale_snowfall_rate_water_equivalent',
'friction_velocity',
'temperature_of_snow_layer',
'ice_temperature_layer_1', 'ice_temperature_layer_2',
'ice_temperature_layer_3', 'ice_temperature_layer_4',
'mean_sea_level_pressure', 'surface_pressure',
'near_ir_albedo_for_diffuse_radiation',
'near_ir_albedo_for_direct_radiation',
'snow_albedo',
'orography',
'precipitation_type',
'snow_density', 'snow_depth',
'snow_evaporation', 'snowfall', 'snowmelt',
'surface_latent_heat_flux', 'surface_sensible_heat_flux',
'surface_net_solar_radiation', 'surface_net_solar_radiation_clear_sky',
'surface_net_thermal_radiation',
'surface_net_thermal_radiation_clear_sky',
'surface_solar_radiation_downward_clear_sky',
'surface_solar_radiation_downwards',
'surface_thermal_radiation_downward_clear_sky',
'surface_thermal_radiation_downwards',
'total_column_cloud_ice_water', 'total_column_cloud_liquid_water',
'total_column_rain_water', 'total_column_snow_water',
'total_column_supercooled_liquid_water', 'total_column_water',
'total_column_water_vapour', 'total_precipitation',
'total_column_ozone',
],
'year': [str(year)],
'month': [month],
'day': [day],
'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00',
'08:00', '09:00', '10:00', '11:00', '12:00', '13:00', '14:00', '15:00',
'16:00', '17:00', '18:00', '19:00', '20:00', '21:00', '22:00', '23:00', ],
'area': [73, -39.5, 72, -37.5, ],
}, file_name)
except Exception:
continue
def download_era5_data(year, month, days, hours, data_path):
"""Downloads ERA5 data for the specified days and hours in a given year/month.
PARAMETERS:
-----------
year: Year of ERA5 data (four-character string)
month: Month of ERA5 data (two-character string)
days: List of days in year/month for which ERA5 data is desired (list of
two-character strings)
hours: List of hours for which ERA5 data is desired (list of two-character
strings)
data_path: Path to main directory for data storage. Specified in
global_variables.py.
OUTPUTS:
--------
None.
This function will alwayd download the same variables these variables can be
adjusted inside the function.
"""
c = cdsapi.Client()
for day in days:
atmos_fname = os.path.join(
data_path, "ERA5",
"ERA5_atmospheric_vbls_" + year + month + day + ".nc")
slvl_fname = os.path.join(
data_path, "ERA5",
"ERA5_singlelevel_vbls_" + year + month + day + ".nc")
c.retrieve(
"reanalysis-era5-pressure-levels", {
"product_type":
"reanalysis",
"format":
"netcdf",
"area":
"90.00/-180.00/60.00/180.00",
"variable": [
'geopotential', 'specific_cloud_ice_water_content',
'specific_cloud_liquid_water_content', 'specific_humidity',
'temperature', 'u_component_of_wind',
'v_component_of_wind', 'vertical_velocity'
],
"pressure_level": ["850", "700"],
"year":
year,
"month":
month,
"day":
day,
"time":
hours
}, atmos_fname)
c.retrieve(
'reanalysis-era5-single-levels', {
'product_type':
'reanalysis',
'format':
'netcdf',
'area':
'90.00/-180.00/60.00/180.00',
'variable': [
'2m_dewpoint_temperature', '2m_temperature',
'mean_sea_level_pressure', 'surface_pressure'
],
'year':
year,
'month':
month,
'day':
day,
'time':
hours
}, slvl_fname)
return None
def main():
# 1. ERA 20C 1900-1979
create_date = ''
years = np.arange(1900,1980)
mnth = np.arange(1,13)
for yr in range(len(years)):
for mn in range(len(mnth)):
create_date = create_date+(str(years[yr]) + str(mnth[mn]).zfill(2)+'01/')
create_date = create_date[:-1]
server = ECMWFDataServer()
server.retrieve({
"class": "e2",
"dataset": "era20c",
"date": create_date,
"expver": "1",
"levtype": "sfc",
"param": "151.128/165.128/166.128",
"stream": "moda",
"type": "an",
"target": "output",
'grid': "1/1",
'format' : "netcdf"
})
# ERA 5 1980-2018
c = cdsapi.Client()
data = c.retrieve(
'reanalysis-era5-single-levels-monthly-means',
{
'product_type':'monthly_averaged_reanalysis',
'variable':[
'10m_u_component_of_wind','10m_v_component_of_wind','mean_sea_level_pressure'
],
'year':[
'1979','1980','1981',
'1982','1983','1984',
'1985','1986','1987',
'1988','1989','1990',
'1991','1992','1993',
'1994','1995','1996',
'1997','1998','1999',
'2000','2001','2002',
'2003','2004','2005',
'2006','2007','2008',
'2009','2010','2011',
'2012','2013','2014',
'2015','2016','2017',
'2018'
],
'month':[
'01','02','03',
'04','05','06',
'07','08','09',
'10','11','12'
],
'time':'00:00',
'format':'netcdf',
'grid': "1/1",
},"/Users/tfrederi/Downloads/ERA5.nc")
return
def call_era5t(output_path: str, year: str, month: str, day: str) -> None:
"""Call cdpaspi to get ERA5T data as file nc format for given date.
Most recent available data is Day -5.
By default "time" = "14:00". It is not an issue since we get these ERA5T data
with a 5 days delay.
Args:
output_path: str
year: str
month: str
day: str
"""
file_path = os.path.join(output_path, f"era5t_{year}_{month}_{day}.nc")
if os.path.exists(file_path):
logger.info(f"Using cached {file_path}")
return
c = cdsapi.Client(url=cfg.CDS_URL,
key=f"{cfg.CDS_UID}:{cfg.CDS_API_KEY}",
verify=0)
c.retrieve(
"reanalysis-era5-single-levels",
{
"product_type":
"reanalysis",
"variable": [
"100m_u_component_of_wind",
"100m_v_component_of_wind",
"10m_u_component_of_neutral_wind",
"10m_u_component_of_wind",
"10m_v_component_of_neutral_wind",
"10m_v_component_of_wind",
"10m_wind_gust_since_previous_post_processing",
"2m_dewpoint_temperature",
"2m_temperature",
"air_density_over_the_oceans",
"altimeter_corrected_wave_height",
"altimeter_range_relative_correction",
"altimeter_wave_height",
"angle_of_sub_gridscale_orography",
"anisotropy_of_sub_gridscale_orography",
"benjamin_feir_index",
"boundary_layer_dissipation",
"boundary_layer_height",
"charnock",
"clear_sky_direct_solar_radiation_at_surface",
"cloud_base_height",
"coefficient_of_drag_with_waves",
"convective_available_potential_energy",
"convective_inhibition",
"convective_precipitation",
"convective_rain_rate",
"convective_snowfall",
"convective_snowfall_rate_water_equivalent",
"downward_uv_radiation_at_the_surface",
"duct_base_height",
"eastward_gravity_wave_surface_stress",
"eastward_turbulent_surface_stress",
"evaporation",
"forecast_albedo",
"forecast_logarithm_of_surface_roughness_for_heat",
"forecast_surface_roughness",
"free_convective_velocity_over_the_oceans",
"friction_velocity",
"gravity_wave_dissipation",
"high_cloud_cover",
"high_vegetation_cover",
"ice_temperature_layer_1",
"ice_temperature_layer_2",
"ice_temperature_layer_3",
"ice_temperature_layer_4",
"instantaneous_10m_wind_gust",
"instantaneous_eastward_turbulent_surface_stress",
"instantaneous_large_scale_surface_precipitation_fraction",
"instantaneous_moisture_flux",
"instantaneous_northward_turbulent_surface_stress",
"instantaneous_surface_sensible_heat_flux",
"k_index",
"lake_bottom_temperature",
"lake_cover",
"lake_depth",
"lake_ice_depth",
"lake_ice_temperature",
"lake_mix_layer_depth",
"lake_mix_layer_temperature",
"lake_shape_factor",
"lake_total_layer_temperature",
"land_sea_mask",
"large_scale_precipitation",
"large_scale_precipitation_fraction",
"large_scale_rain_rate",
"large_scale_snowfall",
"large_scale_snowfall_rate_water_equivalent",
"leaf_area_index_high_vegetation",
"leaf_area_index_low_vegetation",
"low_cloud_cover",
"low_vegetation_cover",
"maximum_2m_temperature_since_previous_post_processing",
"maximum_individual_wave_height",
"maximum_total_precipitation_rate_since_previous_post_processing",
"mean_boundary_layer_dissipation",
"mean_convective_precipitation_rate",
"mean_convective_snowfall_rate",
"mean_direction_of_total_swell",
"mean_direction_of_wind_waves",
"mean_eastward_gravity_wave_surface_stress",
"mean_eastward_turbulent_surface_stress",
"mean_evaporation_rate",
"mean_gravity_wave_dissipation",
"mean_large_scale_precipitation_fraction",
"mean_large_scale_precipitation_rate",
"mean_large_scale_snowfall_rate",
"mean_northward_gravity_wave_surface_stress",
"mean_northward_turbulent_surface_stress",
"mean_period_of_total_swell",
"mean_period_of_wind_waves",
"mean_potential_evaporation_rate",
"mean_runoff_rate",
"mean_sea_level_pressure",
"mean_snow_evaporation_rate",
"mean_snowfall_rate",
"mean_snowmelt_rate",
"mean_square_slope_of_waves",
"mean_sub_surface_runoff_rate",
"mean_surface_direct_short_wave_radiation_flux",
"mean_surface_direct_short_wave_radiation_flux_clear_sky",
"mean_surface_downward_long_wave_radiation_flux",
"mean_surface_downward_long_wave_radiation_flux_clear_sky",
"mean_surface_downward_short_wave_radiation_flux",
"mean_surface_downward_short_wave_radiation_flux_clear_sky",
"mean_surface_downward_uv_radiation_flux",
"mean_surface_latent_heat_flux",
"mean_surface_net_long_wave_radiation_flux",
"mean_surface_net_long_wave_radiation_flux_clear_sky",
"mean_surface_net_short_wave_radiation_flux",
"mean_surface_net_short_wave_radiation_flux_clear_sky",
"mean_surface_runoff_rate",
"mean_surface_sensible_heat_flux",
"mean_top_downward_short_wave_radiation_flux",
"mean_top_net_long_wave_radiation_flux",
"mean_top_net_long_wave_radiation_flux_clear_sky",
"mean_top_net_short_wave_radiation_flux",
"mean_top_net_short_wave_radiation_flux_clear_sky",
"mean_total_precipitation_rate",
"mean_vertical_gradient_of_refractivity_inside_trapping_layer",
"mean_vertically_integrated_moisture_divergence",
"mean_wave_direction",
"mean_wave_direction_of_first_swell_partition",
"mean_wave_direction_of_second_swell_partition",
"mean_wave_direction_of_third_swell_partition",
"mean_wave_period",
"mean_wave_period_based_on_first_moment",
"mean_wave_period_based_on_first_moment_for_swell",
"mean_wave_period_based_on_first_moment_for_wind_waves",
"mean_wave_period_based_on_second_moment_for_swell",
"mean_wave_period_based_on_second_moment_for_wind_waves",
"mean_wave_period_of_first_swell_partition",
"mean_wave_period_of_second_swell_partition",
"mean_wave_period_of_third_swell_partition",
"mean_zero_crossing_wave_period",
"medium_cloud_cover",
"minimum_2m_temperature_since_previous_post_processing",
"minimum_total_precipitation_rate_since_previous_post_processing",
"minimum_vertical_gradient_of_refractivity_inside_trapping_layer",
"model_bathymetry",
"near_ir_albedo_for_diffuse_radiation",
"near_ir_albedo_for_direct_radiation",
"normalized_energy_flux_into_ocean",
"normalized_energy_flux_into_waves",
"normalized_stress_into_ocean",
"northward_gravity_wave_surface_stress",
"northward_turbulent_surface_stress",
"ocean_surface_stress_equivalent_10m_neutral_wind_direction",
"ocean_surface_stress_equivalent_10m_neutral_wind_speed",
"orography",
"peak_wave_period",
"period_corresponding_to_maximum_individual_wave_height",
"potential_evaporation",
"precipitation_type",
"runoff",
"sea_ice_cover",
"sea_surface_temperature",
"significant_height_of_combined_wind_waves_and_swell",
"significant_height_of_total_swell",
"significant_height_of_wind_waves",
"significant_wave_height_of_first_swell_partition",
"significant_wave_height_of_second_swell_partition",
"significant_wave_height_of_third_swell_partition",
"skin_reservoir_content",
"skin_temperature",
"slope_of_sub_gridscale_orography",
"snow_albedo",
"snow_density",
"snow_depth",
"snow_evaporation",
"snowfall",
"snowmelt",
"soil_temperature_level_1",
"soil_temperature_level_2",
"soil_temperature_level_3",
"soil_temperature_level_4",
"soil_type",
"standard_deviation_of_filtered_subgrid_orography",
"standard_deviation_of_orography",
"sub_surface_runoff",
"surface_latent_heat_flux",
"surface_net_solar_radiation",
"surface_net_solar_radiation_clear_sky",
"surface_net_thermal_radiation",
"surface_net_thermal_radiation_clear_sky",
"surface_pressure",
"surface_runoff",
"surface_sensible_heat_flux",
"surface_solar_radiation_downward_clear_sky",
"surface_solar_radiation_downwards",
"surface_thermal_radiation_downward_clear_sky",
"surface_thermal_radiation_downwards",
"temperature_of_snow_layer",
"toa_incident_solar_radiation",
"top_net_solar_radiation",
"top_net_solar_radiation_clear_sky",
"top_net_thermal_radiation",
"top_net_thermal_radiation_clear_sky",
"total_cloud_cover",
"total_column_cloud_ice_water",
"total_column_cloud_liquid_water",
"total_column_ozone",
"total_column_rain_water",
"total_column_snow_water",
"total_column_supercooled_liquid_water",
"total_column_water",
"total_column_water_vapour",
"total_precipitation",
"total_sky_direct_solar_radiation_at_surface",
"total_totals_index",
"trapping_layer_base_height",
"trapping_layer_top_height",
"type_of_high_vegetation",
"type_of_low_vegetation",
"u_component_stokes_drift",
"uv_visible_albedo_for_diffuse_radiation",
"uv_visible_albedo_for_direct_radiation",
"v_component_stokes_drift",
"vertical_integral_of_divergence_of_cloud_frozen_water_flux",
"vertical_integral_of_divergence_of_cloud_liquid_water_flux",
"vertical_integral_of_divergence_of_geopotential_flux",
"vertical_integral_of_divergence_of_kinetic_energy_flux",
"vertical_integral_of_divergence_of_mass_flux",
"vertical_integral_of_divergence_of_moisture_flux",
"vertical_integral_of_divergence_of_ozone_flux",
"vertical_integral_of_divergence_of_thermal_energy_flux",
"vertical_integral_of_divergence_of_total_energy_flux",
"vertical_integral_of_eastward_cloud_frozen_water_flux",
"vertical_integral_of_eastward_cloud_liquid_water_flux",
"vertical_integral_of_eastward_geopotential_flux",
"vertical_integral_of_eastward_heat_flux",
"vertical_integral_of_eastward_kinetic_energy_flux",
"vertical_integral_of_eastward_mass_flux",
"vertical_integral_of_eastward_ozone_flux",
"vertical_integral_of_eastward_total_energy_flux",
"vertical_integral_of_eastward_water_vapour_flux",
"vertical_integral_of_energy_conversion",
"vertical_integral_of_kinetic_energy",
"vertical_integral_of_mass_of_atmosphere",
"vertical_integral_of_mass_tendency",
"vertical_integral_of_northward_cloud_frozen_water_flux",
"vertical_integral_of_northward_cloud_liquid_water_flux",
"vertical_integral_of_northward_geopotential_flux",
"vertical_integral_of_northward_heat_flux",
"vertical_integral_of_northward_kinetic_energy_flux",
"vertical_integral_of_northward_mass_flux",
"vertical_integral_of_northward_ozone_flux",
"vertical_integral_of_northward_total_energy_flux",
"vertical_integral_of_northward_water_vapour_flux",
"vertical_integral_of_potential_and_internal_energy",
"vertical_integral_of_potential_internal_and_latent_energy",
"vertical_integral_of_temperature",
"vertical_integral_of_thermal_energy",
"vertical_integral_of_total_energy",
"vertically_integrated_moisture_divergence",
"volumetric_soil_water_layer_1",
"volumetric_soil_water_layer_2",
"volumetric_soil_water_layer_3",
"volumetric_soil_water_layer_4",
"wave_spectral_directional_width",
"wave_spectral_directional_width_for_swell",
"wave_spectral_directional_width_for_wind_waves",
"wave_spectral_kurtosis",
"wave_spectral_peakedness",
"wave_spectral_skewness",
"zero_degree_level",
],
"year":
year,
"month":
month,
"day":
day,
"time":
"14:00",
"area": [
51,
-6,
41,
10,
],
"format":
"netcdf",
},
file_path,
)
def run(self):
"""Run method that performs all the real work"""
if self.first_start == True:
self.first_start = False
self.dlg = ERA5DataReaderDialog()
self.help = HelpDialog()
self.about = AboutDialog()
self.dlg.pushButton.clicked.connect(self.select_output_file)
self.dlg.About.clicked.connect(self.about_clicked)
self.dlg.button_box.helpRequested.connect(self.help_clicked)
self.dlg.SelectAllMonths.clicked.connect(self.SelectAllMonths)
self.dlg.ClearAllMonths.clicked.connect(self.ClearAllMonths)
self.dlg.SelectAllDays.clicked.connect(self.SelectAllDays)
self.dlg.ClearAllDays.clicked.connect(self.ClearAllDays)
self.dlg.SelectAllTime.clicked.connect(self.SelectAllTime)
self.dlg.ClearAllTime.clicked.connect(self.ClearAllTime)
self.dlg.SelectAllMonths.clicked.connect(self.clear_month_text)
self.dlg.ClearAllMonths.clicked.connect(self.change_month_text)
self.dlg.SelectAllDays.clicked.connect(self.clear_day_text)
self.dlg.ClearAllDays.clicked.connect(self.change_day_text)
self.dlg.SelectAllTime.clicked.connect(self.clear_time_text)
self.dlg.ClearAllTime.clicked.connect(self.change_time_text)
self.dlg.January.stateChanged.connect(self.clear_month_text)
self.dlg.February.stateChanged.connect(self.clear_month_text)
self.dlg.March.stateChanged.connect(self.clear_month_text)
self.dlg.April.stateChanged.connect(self.clear_month_text)
self.dlg.May.stateChanged.connect(self.clear_month_text)
self.dlg.June.stateChanged.connect(self.clear_month_text)
self.dlg.July.stateChanged.connect(self.clear_month_text)
self.dlg.August.stateChanged.connect(self.clear_month_text)
self.dlg.September.stateChanged.connect(self.clear_month_text)
self.dlg.October.stateChanged.connect(self.clear_month_text)
self.dlg.November.stateChanged.connect(self.clear_month_text)
self.dlg.December.stateChanged.connect(self.clear_month_text)
self.dlg.Day1.stateChanged.connect(self.clear_day_text)
self.dlg.Day2.stateChanged.connect(self.clear_day_text)
self.dlg.Day3.stateChanged.connect(self.clear_day_text)
self.dlg.Day4.stateChanged.connect(self.clear_day_text)
self.dlg.Day5.stateChanged.connect(self.clear_day_text)
self.dlg.Day6.stateChanged.connect(self.clear_day_text)
self.dlg.Day7.stateChanged.connect(self.clear_day_text)
self.dlg.Day8.stateChanged.connect(self.clear_day_text)
self.dlg.Day9.stateChanged.connect(self.clear_day_text)
self.dlg.Day10.stateChanged.connect(self.clear_day_text)
self.dlg.Day11.stateChanged.connect(self.clear_day_text)
self.dlg.Day12.stateChanged.connect(self.clear_day_text)
self.dlg.Day13.stateChanged.connect(self.clear_day_text)
self.dlg.Day14.stateChanged.connect(self.clear_day_text)
self.dlg.Day15.stateChanged.connect(self.clear_day_text)
self.dlg.Day16.stateChanged.connect(self.clear_day_text)
self.dlg.Day17.stateChanged.connect(self.clear_day_text)
self.dlg.Day18.stateChanged.connect(self.clear_day_text)
self.dlg.Day19.stateChanged.connect(self.clear_day_text)
self.dlg.Day20.stateChanged.connect(self.clear_day_text)
self.dlg.Day21.stateChanged.connect(self.clear_day_text)
self.dlg.Day22.stateChanged.connect(self.clear_day_text)
self.dlg.Day23.stateChanged.connect(self.clear_day_text)
self.dlg.Day24.stateChanged.connect(self.clear_day_text)
self.dlg.Day25.stateChanged.connect(self.clear_day_text)
self.dlg.Day26.stateChanged.connect(self.clear_day_text)
self.dlg.Day27.stateChanged.connect(self.clear_day_text)
self.dlg.Day28.stateChanged.connect(self.clear_day_text)
self.dlg.Day29.stateChanged.connect(self.clear_day_text)
self.dlg.Day30.stateChanged.connect(self.clear_day_text)
self.dlg.Day31.stateChanged.connect(self.clear_day_text)
self.dlg.Time0.stateChanged.connect(self.clear_time_text)
self.dlg.Time1.stateChanged.connect(self.clear_time_text)
self.dlg.Time2.stateChanged.connect(self.clear_time_text)
self.dlg.Time3.stateChanged.connect(self.clear_time_text)
self.dlg.Time4.stateChanged.connect(self.clear_time_text)
self.dlg.Time5.stateChanged.connect(self.clear_time_text)
self.dlg.Time6.stateChanged.connect(self.clear_time_text)
self.dlg.Time7.stateChanged.connect(self.clear_time_text)
self.dlg.Time8.stateChanged.connect(self.clear_time_text)
self.dlg.Time9.stateChanged.connect(self.clear_time_text)
self.dlg.Time10.stateChanged.connect(self.clear_time_text)
self.dlg.Time11.stateChanged.connect(self.clear_time_text)
self.dlg.Time12.stateChanged.connect(self.clear_time_text)
self.dlg.Time13.stateChanged.connect(self.clear_time_text)
self.dlg.Time14.stateChanged.connect(self.clear_time_text)
self.dlg.Time15.stateChanged.connect(self.clear_time_text)
self.dlg.Time16.stateChanged.connect(self.clear_time_text)
self.dlg.Time17.stateChanged.connect(self.clear_time_text)
self.dlg.Time18.stateChanged.connect(self.clear_time_text)
self.dlg.Time19.stateChanged.connect(self.clear_time_text)
self.dlg.Time20.stateChanged.connect(self.clear_time_text)
self.dlg.Time21.stateChanged.connect(self.clear_time_text)
self.dlg.Time22.stateChanged.connect(self.clear_time_text)
self.dlg.Time23.stateChanged.connect(self.clear_time_text)
self.dlg.selectonemonth.clear()
self.dlg.selectoneday.clear()
self.dlg.selectonetime.clear()
#Set validator for years
regexYear = QRegExp("[0-9_\s]+")
validatorYear = QRegExpValidator(regexYear)
self.dlg.Year.setValidator(validatorYear)
#Set validator for coordinates
validatorNorth = QDoubleValidator(-90, 90, 2)
validatorNorth.setNotation(QDoubleValidator.StandardNotation)
validatorWest = QDoubleValidator(-180, 180, 2)
validatorWest.setNotation(QDoubleValidator.StandardNotation)
validatorSouth = QDoubleValidator(-90, 90, 2)
validatorSouth.setNotation(QDoubleValidator.StandardNotation)
validatorEast = QDoubleValidator(-180, 180, 2)
validatorEast.setNotation(QDoubleValidator.StandardNotation)
self.dlg.NorthCoordinate.setValidator(validatorNorth)
self.dlg.WestCoordinate.setValidator(validatorWest)
self.dlg.SouthCoordinate.setValidator(validatorSouth)
self.dlg.EastCoordinate.setValidator(validatorEast)
self.dlg.show()
result = self.dlg.exec_()
Years = list()
Months = list()
Days = list()
Time = list()
Area = list()
if result:
#Append selected checkboxes in the corresponding list
if self.dlg.January.isChecked():
Months.append('01')
if self.dlg.February.isChecked():
Months.append('02')
if self.dlg.March.isChecked():
Months.append('03')
if self.dlg.April.isChecked():
Months.append('04')
if self.dlg.May.isChecked():
Months.append('05')
if self.dlg.June.isChecked():
Months.append('06')
if self.dlg.July.isChecked():
Months.append('07')
if self.dlg.August.isChecked():
Months.append('08')
if self.dlg.September.isChecked():
Months.append('09')
if self.dlg.October.isChecked():
Months.append('10')
if self.dlg.November.isChecked():
Months.append('11')
if self.dlg.December.isChecked():
Months.append('12')
if self.dlg.Day1.isChecked():
Days.append('01')
if self.dlg.Day2.isChecked():
Days.append('02')
if self.dlg.Day3.isChecked():
Days.append('03')
if self.dlg.Day4.isChecked():
Days.append('04')
if self.dlg.Day5.isChecked():
Days.append('05')
if self.dlg.Day6.isChecked():
Days.append('06')
if self.dlg.Day7.isChecked():
Days.append('07')
if self.dlg.Day8.isChecked():
Days.append('08')
if self.dlg.Day9.isChecked():
Days.append('09')
if self.dlg.Day10.isChecked():
Days.append('10')
if self.dlg.Day11.isChecked():
Days.append('11')
if self.dlg.Day12.isChecked():
Days.append('12')
if self.dlg.Day13.isChecked():
Days.append('13')
if self.dlg.Day14.isChecked():
Days.append('14')
if self.dlg.Day15.isChecked():
Days.append('15')
if self.dlg.Day16.isChecked():
Days.append('16')
if self.dlg.Day17.isChecked():
Days.append('17')
if self.dlg.Day18.isChecked():
Days.append('18')
if self.dlg.Day19.isChecked():
Days.append('19')
if self.dlg.Day20.isChecked():
Days.append('20')
if self.dlg.Day21.isChecked():
Days.append('21')
if self.dlg.Day22.isChecked():
Days.append('22')
if self.dlg.Day23.isChecked():
Days.append('23')
if self.dlg.Day24.isChecked():
Days.append('24')
if self.dlg.Day25.isChecked():
Days.append('25')
if self.dlg.Day26.isChecked():
Days.append('26')
if self.dlg.Day27.isChecked():
Days.append('27')
if self.dlg.Day28.isChecked():
Days.append('28')
if self.dlg.Day29.isChecked():
Days.append('29')
if self.dlg.Day30.isChecked():
Days.append('30')
if self.dlg.Day31.isChecked():
Days.append('31')
if self.dlg.January.isChecked():
Months.append('01')
if self.dlg.February.isChecked():
Months.append('02')
if self.dlg.March.isChecked():
Months.append('03')
if self.dlg.April.isChecked():
Months.append('04')
if self.dlg.May.isChecked():
Months.append('05')
if self.dlg.June.isChecked():
Months.append('06')
if self.dlg.July.isChecked():
Months.append('07')
if self.dlg.August.isChecked():
Months.append('08')
if self.dlg.September.isChecked():
Months.append('09')
if self.dlg.October.isChecked():
Months.append('10')
if self.dlg.November.isChecked():
Months.append('11')
if self.dlg.December.isChecked():
Months.append('12')
if self.dlg.Time0.isChecked():
Time.append('00:00')
if self.dlg.Time1.isChecked():
Time.append('01:00')
if self.dlg.Time2.isChecked():
Time.append('02:00')
if self.dlg.Time3.isChecked():
Time.append('03:00')
if self.dlg.Time4.isChecked():
Time.append('04:00')
if self.dlg.Time5.isChecked():
Time.append('05:00')
if self.dlg.Time6.isChecked():
Time.append('06:00')
if self.dlg.Time7.isChecked():
Time.append('07:00')
if self.dlg.Time8.isChecked():
Time.append('08:00')
if self.dlg.Time9.isChecked():
Time.append('09:00')
if self.dlg.Time10.isChecked():
Time.append('10:00')
if self.dlg.Time11.isChecked():
Time.append('11:00')
if self.dlg.Time12.isChecked():
Time.append('12:00')
if self.dlg.Time13.isChecked():
Time.append('13:00')
if self.dlg.Time14.isChecked():
Time.append('14:00')
if self.dlg.Time15.isChecked():
Time.append('15:00')
if self.dlg.Time16.isChecked():
Time.append('16:00')
if self.dlg.Time17.isChecked():
Time.append('17:00')
if self.dlg.Time18.isChecked():
Time.append('18:00')
if self.dlg.Time19.isChecked():
Time.append('19:00')
if self.dlg.Time20.isChecked():
Time.append('20:00')
if self.dlg.Time21.isChecked():
Time.append('21:00')
if self.dlg.Time22.isChecked():
Time.append('22:00')
if self.dlg.Time23.isChecked():
Time.append('23:00')
#Append years in a list
for Y in self.dlg.Year.text().split(' '):
Years.append(Y)
#Append coordinates in a list
for North in self.dlg.NorthCoordinate.text().split(' '):
NorthInt = float(North)
Area.append(NorthInt)
for West in self.dlg.WestCoordinate.text().split(' '):
WestInt = float(West)
Area.append(WestInt)
for South in self.dlg.SouthCoordinate.text().split(' '):
SouthInt = float(South)
Area.append(SouthInt)
for East in self.dlg.EastCoordinate.text().split(' '):
EastInt = float(East)
Area.append(EastInt)
try:
if filename == "":
Layer_Name = self.dlg.LayerName.text()
pn = QgsProject.instance().readPath("./")
sl = '/'
fn = Layer_Name + ".nc"
selectpath = False
else:
originalpath = QgsProject.instance().readPath("./")
Layer_Name = re.findall('(^.*).nc',
self.dlg.LayerName.text())[0]
pn = re.findall('^.*/', self.dlg.LayerName.text())[0]
sl = '/'
fn = re.findall('^.*/(\S+)', self.dlg.LayerName.text())[0]
selectpath = True
except:
Layer_Name = self.dlg.LayerName.text()
pn = QgsProject.instance().readPath("./")
sl = '/'
fn = Layer_Name + ".nc"
selectpath = False
#Download evaporation data through the API
c = cdsapi.Client()
c.retrieve(
'reanalysis-era5-single-levels', {
'product_type': 'reanalysis',
'variable': 'mean_evaporation_rate',
'year': Years,
'month': Months,
'day': Days,
'time': Time,
'area': Area,
'format': 'netcdf',
}, fn)
if selectpath == True:
original = originalpath + sl + fn
target = pn + sl + fn
shutil.move(original, target)
rlayer = iface.addRasterLayer(target, '')
nbands = rlayer.bandCount()
print(fn, nbands, "bands")
else:
rlayer = iface.addRasterLayer(pn + sl + fn, '')
nbands = rlayer.bandCount()
print(fn, nbands, "bands")
#Show download success message
self.iface.messageBar().pushMessage("Success",
"Output file downloaded at " +
pn + fn,
level=Qgis.Success,
duration=8)
#Append band values in an array
wd = rlayer.width()
ht = rlayer.height()
values = []
provider = rlayer.dataProvider()
for aband in range(rlayer.bandCount()):
block = provider.block(aband + 1, rlayer.extent(), wd, ht)
for i in range(rlayer.width()):
for j in range(rlayer.height()):
values.append(block.value(i, j))
arr = np.reshape(array(values), (-1, wd, ht))
arrnew = np.moveaxis(arr, 0, -1)
arrnew = -arrnew * 60 * 60
#Carry out statistical anaylysis and place each parameter for each pixel in an array
arrsum = []
arravg = []
arrmax = []
arrmin = []
for arr1 in arrnew:
for arr2 in arr1:
arrsum.append(np.sum(arr2))
arravg.append(np.average(arr2))
arrmax.append(np.max(arr2))
arrmin.append(np.min(arr2))
rearrsum = np.reshape(array(arrsum),
(rlayer.width(), rlayer.height()))
#Create arrays for row and column count
arrrow = []
arrcol = []
ro = -1
p = wd * ht
while len(arrrow) < p:
ro = ro + 1
co = -1
for r in range(wd):
arrrow.append(ro)
co = co + 1
arrcol.append(co)
#Append array values into a dataframe
dfrow = pd.DataFrame(arrrow, columns=["Row"])
dfcol = pd.DataFrame(arrcol, columns=["Column"])
dfsum = pd.DataFrame(arrsum, columns=["Sum"])
dfavg = pd.DataFrame(arravg, columns=["Average"])
dfmax = pd.DataFrame(arrmax, columns=["Max"])
dfmin = pd.DataFrame(arrmin, columns=["Min"])
df1 = pd.merge(dfrow,
dfcol,
how="outer",
left_index=True,
right_index=True)
df2 = pd.merge(df1,
dfsum,
how="outer",
left_index=True,
right_index=True)
df3 = pd.merge(df2,
dfavg,
how="outer",
left_index=True,
right_index=True)
df4 = pd.merge(df3,
dfmax,
how="outer",
left_index=True,
right_index=True)
dfin = pd.merge(df4,
dfmin,
how="outer",
left_index=True,
right_index=True)
#Create point vector layer having statistical data
alg_params = {
'FIELD_NAME': 'VALUE',
'INPUT_RASTER': rlayer,
'RASTER_BAND': 1,
'OUTPUT': QgsProcessing.TEMPORARY_OUTPUT
}
vlay = processing.run('qgis:pixelstopoints', alg_params)
vlayer = vlay['OUTPUT']
vlayer.startEditing()
for head in dfin:
myField = QgsField(head, QVariant.Double)
vlayer.addAttribute(myField)
provider = rlayer.dataProvider()
band1 = []
for i in range(rlayer.height()):
for j in range(rlayer.width()):
block = provider.block(1, rlayer.extent(), rlayer.width(),
rlayer.height())
band1.append(block.value(i, j))
x = 0
for feature in vlayer.getFeatures():
att = feature.attributes()[0]
if att == band1[x]:
feature[1] = int(dfin.iloc[x]['Row'])
feature[2] = int(dfin.iloc[x]['Column'])
feature[3] = str(dfin.iloc[x]['Sum'])
feature[4] = str(dfin.iloc[x]['Average'])
feature[5] = str(dfin.iloc[x]['Max'])
feature[6] = str(dfin.iloc[x]['Min'])
vlayer.updateFeature(feature)
x = x + 1
vlayer.commitChanges()
vlayer.dataProvider().deleteAttributes([0])
vlayer.updateFields()
QgsProject.instance().addMapLayer(vlayer)
#Show vector points layer success message
self.iface.messageBar().pushMessage("Success",
"Vector points layer loaded.",
level=Qgis.Success,
duration=4)
#Reset conditions
self.dlg.Year.clear()
self.dlg.LayerName.clear()
self.dlg.NorthCoordinate.clear()
self.dlg.WestCoordinate.clear()
self.dlg.SouthCoordinate.clear()
self.dlg.EastCoordinate.clear()
self.dlg.selectonemonth.clear()
self.dlg.selectoneday.clear()
self.dlg.selectonetime.clear()
self.dlg.January.setChecked(False)
self.dlg.February.setChecked(False)
self.dlg.March.setChecked(False)
self.dlg.April.setChecked(False)
self.dlg.May.setChecked(False)
self.dlg.June.setChecked(False)
self.dlg.July.setChecked(False)
self.dlg.August.setChecked(False)
self.dlg.September.setChecked(False)
self.dlg.October.setChecked(False)
self.dlg.November.setChecked(False)
self.dlg.December.setChecked(False)
self.dlg.Day1.setChecked(False)
self.dlg.Day2.setChecked(False)
self.dlg.Day3.setChecked(False)
self.dlg.Day4.setChecked(False)
self.dlg.Day5.setChecked(False)
self.dlg.Day6.setChecked(False)
self.dlg.Day7.setChecked(False)
self.dlg.Day8.setChecked(False)
self.dlg.Day9.setChecked(False)
self.dlg.Day10.setChecked(False)
self.dlg.Day11.setChecked(False)
self.dlg.Day12.setChecked(False)
self.dlg.Day13.setChecked(False)
self.dlg.Day14.setChecked(False)
self.dlg.Day15.setChecked(False)
self.dlg.Day16.setChecked(False)
self.dlg.Day17.setChecked(False)
self.dlg.Day18.setChecked(False)
self.dlg.Day19.setChecked(False)
self.dlg.Day20.setChecked(False)
self.dlg.Day21.setChecked(False)
self.dlg.Day21.setChecked(False)
self.dlg.Day22.setChecked(False)
self.dlg.Day23.setChecked(False)
self.dlg.Day24.setChecked(False)
self.dlg.Day25.setChecked(False)
self.dlg.Day26.setChecked(False)
self.dlg.Day27.setChecked(False)
self.dlg.Day28.setChecked(False)
self.dlg.Day29.setChecked(False)
self.dlg.Day30.setChecked(False)
self.dlg.Day31.setChecked(False)
self.dlg.Time0.setChecked(False)
self.dlg.Time1.setChecked(False)
self.dlg.Time2.setChecked(False)
self.dlg.Time3.setChecked(False)
self.dlg.Time4.setChecked(False)
self.dlg.Time5.setChecked(False)
self.dlg.Time6.setChecked(False)
self.dlg.Time7.setChecked(False)
self.dlg.Time8.setChecked(False)
self.dlg.Time9.setChecked(False)
self.dlg.Time10.setChecked(False)
self.dlg.Time11.setChecked(False)
self.dlg.Time12.setChecked(False)
self.dlg.Time13.setChecked(False)
self.dlg.Time14.setChecked(False)
self.dlg.Time15.setChecked(False)
self.dlg.Time16.setChecked(False)
self.dlg.Time17.setChecked(False)
self.dlg.Time18.setChecked(False)
self.dlg.Time19.setChecked(False)
self.dlg.Time20.setChecked(False)
self.dlg.Time21.setChecked(False)
self.dlg.Time22.setChecked(False)
self.dlg.Time23.setChecked(False)
def call_era5land(output_path: str, year: str, month: str, day: str) -> None:
"""Call cdpaspi to get ERA5Land data as file nc format for given date.
By default "time" = "14:00". It is not an issue since we get these ERA5 Land data
with a 2 months delay.
Args:
output_path: str
year: str
month: str
day: str
"""
file_path = os.path.join(output_path, f"era5land_{year}_{month}_{day}.nc")
if os.path.exists(file_path):
logger.info(f"Using cached {file_path}")
return
c = cdsapi.Client(url=cfg.CDS_URL,
key=f"{cfg.CDS_UID}:{cfg.CDS_API_KEY}",
verify=0)
c.retrieve(
"reanalysis-era5-land",
{
"variable": [
"10m_u_component_of_wind",
"10m_v_component_of_wind",
"2m_dewpoint_temperature",
"2m_temperature",
"evaporation_from_bare_soil",
"evaporation_from_open_water_surfaces_excluding_oceans",
"evaporation_from_the_top_of_canopy",
"evaporation_from_vegetation_transpiration",
"forecast_albedo",
"lake_bottom_temperature",
"lake_ice_depth",
"lake_ice_temperature",
"lake_mix_layer_depth",
"lake_mix_layer_temperature",
"lake_shape_factor",
"lake_total_layer_temperature",
"leaf_area_index_high_vegetation",
"leaf_area_index_low_vegetation",
"potential_evaporation",
"runoff",
"skin_reservoir_content",
"skin_temperature",
"snow_albedo",
"snow_cover",
"snow_density",
"snow_depth",
"snow_depth_water_equivalent",
"snow_evaporation",
"snowfall",
"snowmelt",
"soil_temperature_level_1",
"soil_temperature_level_2",
"soil_temperature_level_3",
"soil_temperature_level_4",
"sub_surface_runoff",
"surface_latent_heat_flux",
"surface_net_solar_radiation",
"surface_net_thermal_radiation",
"surface_pressure",
"surface_runoff",
"surface_sensible_heat_flux",
"surface_solar_radiation_downwards",
"surface_thermal_radiation_downwards",
"temperature_of_snow_layer",
"total_evaporation",
"total_precipitation",
"volumetric_soil_water_layer_1",
"volumetric_soil_water_layer_2",
"volumetric_soil_water_layer_3",
"volumetric_soil_water_layer_4",
],
"year":
year,
"month":
month,
"day":
day,
"time":
"14:00",
"area": [
51,
-6,
41,
10,
],
"format":
"netcdf",
},
file_path,
)
def do_request(r):
"""
Issue the download request. param 'r' is a tuple:
[0] dataset name
[1] the query
[2] file staging path
[3] file target path
[4] ip for download url
Download to staging area first, compress netcdf (nccopy)
"""
global cfg, era5log
tempfn = r[2]
fn = r[3]
era5log.debug(f"{tempfn}")
era5log.debug(f"{fn}")
# the actual retrieve part
# create client instance
c = cdsapi.Client()
era5log.info(f'Downloading {tempfn} ... ')
era5log.info(f'Request: {r[1]}')
# need to capture exceptions
apirc = False
try:
# issue the request (to modified api)
res = c.retrieve(r[0], r[1], tempfn)
apirc = True
except Exception as e:
era5log.error(f'ERROR: {e}')
apirc = False
# if request successful download file
if apirc:
# get download url and replace ip
url = res.location
if '.198/' in res.location:
url = res.location.replace('.198/', f'.{r[4]}/')
cmd = f"{cfg['getcmd']} {tempfn} {url}"
era5log.info(f'ERA5 Downloading: {url} to {tempfn}')
#return
p = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = p.communicate()
if not p.returncode: # successful
# do some compression on the file - assuming 1. it's netcdf, 2. that nccopy will fail if file is corrupt
cmd = f"{cfg['nccmd']} {tempfn} {fn}"
p1 = sp.Popen(cmd, shell=True, stdout=sp.PIPE, stderr=sp.PIPE)
out, err = p1.communicate()
if not p1.returncode: # check was successful
era5log.info(f'ERA5 download success: {fn}')
# move to correct destination
#if not os.rename(tempfn, r[3]):
# era5log.info(f'ERA5 moved to {r[3]}')
#
# update db
# ...
else:
era5log.info(
f'ERA5 nc command failed! (deleting temp file {tempfn})\n{err.decode()}'
)
#if not ctx.obj['debug']:
# os.remove(tempfn)
else:
era5log.info(f'ERA5 download error: \n{err.decode()}')
# should try to resume before deleting
#os.remove(tempfn)
return
def download(self):
self.work_dir = os.path.dirname(os.path.abspath(__file__))
self.save_to = os.path.join(
self.work_dir,
'/home/ludo915/code/covsco/data/train/cams/fr/forecast')
if not os.path.exists(self.save_to):
os.makedirs(self.save_to)
# get personal directory of cdsapi
try:
with open('.cdsapirc_cams', 'r') as file:
cams_api = file.readline().rstrip()
except FileNotFoundError:
raise FileNotFoundError("""cdsapirc file cannot be found. Write the
directory of your personal .cdsapirc file in a local file called
`.cdsapirc_cams` and place it in the directory where this script lies."""
)
# Download CAMS
# -----------------------------------------------------------------------------
print('Download data from CAMS ...', flush=True)
with open(cams_api, 'r') as f:
credentials = yaml.safe_load(f)
mypath = "/home/ludo915/code/covsco/data/train/cams/fr/forecast/"
def findlatestdateofcamsdata(mypath):
dates = []
onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
for filename in onlyfiles:
dates.append(pd.to_datetime(filename[14:24]))
if dates != []:
return (dates, max(dates))
else:
return (dates, dt.date.today() - pd.Timedelta("3 Y"))
prevday = dt.date.today() - pd.Timedelta("1 days")
startdate = findlatestdateofcamsdata(mypath)[1]
datesnotclean = pd.date_range(
start=startdate, end=prevday).strftime("%Y-%m-%d").tolist()
dates = []
for date in datesnotclean:
if date not in pd.to_datetime(findlatestdateofcamsdata(mypath)[0]):
dates.append(date)
print(dates)
area = [51.75, -5.83, 41.67, 11.03]
for date in tqdm(dates):
print(date)
file_name = 'cams-forecast-{:}.nc'.format(date)
output_file = os.path.join(self.save_to, file_name)
if not os.path.exists(output_file):
c = cdsapi.Client(url=credentials['url'],
key=credentials['key'])
c.retrieve(
'cams-europe-air-quality-forecasts', {
'variable': [
'carbon_monoxide',
'nitrogen_dioxide',
'ozone',
'particulate_matter_10um',
'particulate_matter_2.5um',
'sulphur_dioxide',
],
'model':
'ensemble',
'level':
'0',
'date':
date,
'type':
'forecast',
'time':
'00:00',
'leadtime_hour': ['0', '24', '48', '72', '96'],
'area':
area,
'format':
'netcdf',
}, output_file)
print('Download finished.', flush=True)