def getDataECMWF(self, filename):
# create a new ECMWFServer object
server = ECMWFDataServer()
#build a dictionary that provides the MARS like information to the ECMWF servers
# start with fixed fields that are the same for all queries
self.MARSCode = {
'stream' : "oper",
'dataset' : "interim",
'param' : str(self.param),
'date' : self.start+"/to/"+self.end,
'area' : str(self.latitude_max)+"/"+str(self.longitude_min)+"/"+str(self.latitude_min)+"/"+str(self.longitude_max),
'time' : "00/12",
'grid' : str(self.resolution)+"/"+str(self.resolution),
'target' : filename,
'levtype' : str(self.levtype),
'type' : str(self.type),
'format' : "netcdf"
}
# if the filed is a forecast, then we want the 12-hour cumulated values.
if self.type is "fc":
self.MARSCode['step'] = "12"
# if the field is analysis then we want the instantaneous values
elif self.type is "an":
self.MARSCode['step'] = '0'
# if the analysed field is also on pressure levels, then define the level that we want.
if self.levtype is "pl":
self.MARSCode['levelist'] = str(self.level)
##retrieve the data
server.retrieve(self.MARSCode)
def retrieve(request):
""" Execute Mars request
Parameters
----------
request
Returns
-------
filename
"""
import os
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
if os.path.isdir(os.path.dirname(request['target'])):
os.makedirs(os.path.isdir(os.path.dirname(request['target'])))
try:
server.retrieve(request)
print "Request was successful."
print request['target']
return True
except Exception as e:
print repr(e)
return False
def fetch_era_data(variable_dict):
""" Download data from ECMWF
Parameters:
-----------
param_dic : dict
A dictionary defining the download, eg:
{"class": "ei",
"dataset": "interim",
"date": '19790101/19790201/19790301',
"grid": "0.75/0.75",
"levtype": "sfc",
"param": '165.128',
"stream": "moda",
"target": 'ERA-Int_slp.mon.mean.nc',
"type": "an",
"format": "netcdf"
}
See ('https://software.ecmwf.int/wiki/display/WEBAPI/' +
'Accessing+ECMWF+data+servers+in+batch')
Returns:
--------
Downloads files to PWD.
"""
server = ECMWFDataServer()
server.retrieve(variable_dict)
def get(parameter, year):
if parameter.isdigit():
try:
paramDict = {row[3]:row[1] for row in csv.reader(open('parameter_info.csv','rU'))}
paramName = paramDict[parameter]
paramID = parameter
except IOError:
print "Parameter ID not valid"
else:
try:
paramDict = {row[1]:row[3] for row in csv.reader(open('parameter_info.csv','rU'))}
paramID = paramDict[parameter]
paramName = parameter
except IOError:
print "Parameter name not valid"
param = paramID + ".128"
outputDir = os.path.expandvars('$HOME/norgay/data/sources/ERAI/3HOURLY/'+paramName+'/')
try:
os.makedirs(outputDir)
except OSError:
if not os.path.isdir(outputDir):
raise
server = ECMWFDataServer()
if year!='1979':
startDate = str(int(year)-1) + '-12-31'
else:
startDate = year + '-01-01'
if year=='2015':
endDate = year + '-06-30'
else:
endDate = year + '-12-31'
targetFile = "ERAI_{}_{}to{}.nc".format(paramName, startDate, endDate)
server.retrieve({
'stream' : "oper",
'levtype' : "sfc",
'param' : param,
'dataset' : "interim",
'step' : "03/06/09/12",
'grid' : "0.25/0.25",
'time' : "00/12",
'date' : "{}/to/{}".format(startDate, endDate),
'type' : "fc",
'class' : "ei",
'format' : "netcdf",
'target' : outputDir+targetFile
})
def API(output_folder, DownloadType, string1, string2, string3, string4, string5, string6, string7, string8, string9, string10):
import os
from ecmwfapi import ECMWFDataServer
os.chdir(output_folder)
server = ECMWFDataServer()
if DownloadType == 1 or DownloadType == 2:
server.retrieve({
'stream' : "%s" %string1,
'levtype' : "%s" %string2,
'param' : "%s" %string3,
'dataset' : "interim",
'step' : "%s" %string4,
'grid' : "%s" %string5,
'time' : "%s" %string6,
'date' : "%s" %string7,
'type' : "%s" %string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
'class' : "%s" %string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
'area' : "%s" %string10,
'format' : "netcdf",
'target' : "data_interim.nc"
})
if DownloadType == 3:
server.retrieve({
'levelist' : "1000",
'stream' : "%s" %string1,
'levtype' : "%s" %string2,
'param' : "%s" %string3,
'dataset' : "interim",
'step' : "%s" %string4,
'grid' : "%s" %string5,
'time' : "%s" %string6,
'date' : "%s" %string7,
'type' : "%s" %string8, # http://apps.ecmwf.int/codes/grib/format/mars/type/
'class' : "%s" %string9, # http://apps.ecmwf.int/codes/grib/format/mars/class/
'area' : "%s" %string10,
'format' : "netcdf",
'target' : "data_interim.nc"
})
return()
def download(year,month):
server = ECMWFDataServer()
file = "/prj/AMMA2050/ERA5/surface/ERA5_"+str(year)+"_"+str(month).zfill(2)+"_srfc.nc"
server.retrieve({
"class": "ea",
"dataset": "era5",
"date": str(year)+"-"+str(month).zfill(2)+"-01/to/"+str(year)+"-"+str(month).zfill(2)+"-31",
"expver": "1",
"grid": "0.25/0.25",
"levtype": "sfc",
"param":"59.128/60.162/61.162/62.162/63.162/71.162/72.162/78.128/79.128/79.162/80.162/81.162/82.162/83.162/84.162/86.162/89.228/129.128/134.128/137.128/151.128/159.128/164.128/165.128/166.128/167.128/168.128/172.128/186.128/187.128/188.128/231.128/232.128/235.128/246.228/247.228" ,
"step": "0",
"stream": "oper",
"time": "06:00:00/09:00:00/12:00:00/15:00:00/18:00:00/21:00:00/00:00:00/03:00:00",
"type": "an",
"area": "25/-18.5/3.5/17",
"format": "netcdf",
"target": file
})
def download(year, month):
server = ECMWFDataServer()
file = "/prj/AMMA2050/ERA5/pressure_levels/ERA5_"+str(year)+"_"+str(month).zfill(2)+"_pl.nc"
server.retrieve({
"class": "ea",
"dataset": "era5",
"date": str(year)+"-"+str(month).zfill(2)+"-01/to/"+str(year)+"-"+str(month).zfill(2)+"-31",
"expver": "1",
"grid": "0.25/0.25",
"levtype": "pl",
"levelist": "200/250/300/350/400/450/500/550/600/650/700/750/825/850/875/900/925/950/975",
"param": "60.128/130.128/131.128/132.128/133.128/135.128/155.128/157.128",
"step": "0",
"stream": "oper",
"time": "06:00:00/09:00:00/12:00:00/15:00:00/18:00:00/21:00:00/00:00:00/03:00:00",
"type": "an",
"area": "25/-18.5/3.5/17",
"format": "netcdf",
"target": file
})
def loop(y):
file = "/localscratch/wllf030/cornkle/ERA-I/daily_"+ str(y) + "_pl_12UTC.nc"
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": str(y)+"-01-01/to/" + str(y) + "-12-31",
"expver": "1",
"grid": "0.75/0.75",
"levtype": "pl",
"levelist": "250/350/450/550/600/650/700/750/800/850/900/925/950",
"param": "60.128/130.128/131.128/132.128/133.128/135.128/155.128/157.128",
"step": "0",
"stream": "oper",
"time": "12:00:00",
"type": "an",
"area": "22/-18/3/15",
"format": "netcdf",
"target": file
})
def retrieve_netcdf(Variables_available,monthlystr,namestr,location):
"""Retrieve NetCDF file"""
from chosen_variables import chosen_variables
from location import chosen_area
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": monthlystr,
"expver": "1",
"grid": "0.125/0.125",
"area": chosen_area(location),
"levtype": "sfc",
"param": chosen_variables(Variables_available),
"step": "0",
"stream": "oper",
"target": namestr,
"time": "00/06/12/18",
"type": "an",
'format' : "netcdf"
})
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": monthlystr,
"expver": "1",
"grid": "0.125/0.125",
"area": chosen_area(location),
"levtype": "sfc",
"param": chosen_variables(Variables_available),
"step": "3/6/9/12",
"stream": "oper",
"target": namestr,
"time": "00/12",
"type": "fc",
'format' : "netcdf"
})
def download_ECMWF(date,days=2):
date2 = date + datetime.timedelta(days=days-1)
datestr = '{0:04d}-{1:02d}-{2:02d}/to/{3:04d}-{4:02d}-{5:02d}'.format(
date.year,date.month,date.day,
date2.year,date2.month,date2.day)
ncstr = 'ECMWF_{0:04d}{1:02d}{2:02d}.nc'.format(
date.year,date.month,date.day)
REQ = {}
REQ['stream'] = 'oper'
REQ['levtype'] = 'pl'
REQ['param'] = 'z/u/v'
REQ['dataset'] = 'interim'
REQ['step'] = '0'
REQ['grid'] = '1/1'
REQ['time'] = '00/06/12/18'
REQ['date'] = datestr
REQ['type'] = 'an'
REQ['class'] = 'ei'
REQ['target'] = os.path.join(ECMWFdir,ncstr)
REQ['format'] = 'netcdf'
server = ECMWFDataServer()
server.retrieve(REQ)
"""
created on July 23, 2014
@author: Nikola Jajcay
"""
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"stream" : "oper",
"levtype" : "pl",
"param" : "60.128/129.128/130.128/131.128/132.128/133.128/135.128/138.128/155.128/157.128/203.128", ## https://badc.nerc.ac.uk/data/ecmwf-e40/params.html
# "dataset" : "interim", ## era40, interim, era20c
"levelist" : "100/300/500/800/850/900/925/950/1000",
"step" : "0",
# "grid" : "2.5/2.5",
"time" : "12:00:00", ## daily
"date" : "2017-04-04",
# "area" : "75/-40/25/80", ## north/west/south/east
"type" : "an", ## an for analysis, fc for forecast
"class" : "od", ## e4 for era40, ei for interim
# "format" : "netcdf",
# "padding" : "0",
"target" : "output", ## filename
"expver" : 1
})
#! /home/meteoboy4/anaconda/bin/python
from ecmwfapi import ECMWFDataServer
#Make a list containing the desired year to download and parsing the filename later
year=["1985"]
for iyear in range(29):
iyear=iyear+1
year.append(str(int(year[0])+iyear))
print(year)
server=ECMWFDataServer()
for iyear in range(len(year)):
server.retrieve({
'stream' : "oper",
'param' : "131.128",
'dataset' : "interim",
'step' : "0",
'grid' : "1.0/1.0",
'levelist' : "300",
'levtype' : "pl",
'time' : "00/06/12/18",
'date' : year[iyear]+"-01-01/to/"+year[iyear]+"-12-31",
'expver' : "1",
'type' : "an",
'class' : "ei",
'format' : "netcdf",
'target' : "/run/media/MeteoBoy4/Data/MData/ERA-Interim/1985-2014_ALEV/Daily4/U_wind_component/300hpa/"+year[iyear]+".nc"
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "2013-01-01/to/2013-01-31",
"expver": "1",
"grid": "0.125/0.125",
"levelist": "400/450/500/550/600/650/700/750/775/800/825/850/875/900/925/950/975/1000",
"levtype": "pl",
"param": "130.128",
"step": "0",
"stream": "oper",
"area": "-2/37/-4/38",
"target": "2013001_2013031__temperature.nc",
"time": "12",
"type": "an",
})
#!/usr/bin/env python26
from ecmwfapi import ECMWFDataServer
import os
def leap_year(year):
if (year % 400 == 0):
x = 1
elif (year % 4 == 0):
x = 1
else:
x = 0
return x
server = ECMWFDataServer()
variables = 'Volumetric soil water layer 3'
out_name = 'SQ3'
mon_name = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'
]
ord_mon = [
'31', '28', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
leap_mon = [
'31', '29', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
for year in range(2014, 2015):
for mon in range(1, 6):
#!/usr/bin/python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
'dataset' : 'interim',
'date' : '20121101/to/20121105',
'stream' : 'oper',
'time' : '00/06/12/18',
'grid' : '0.75/0.75',
'step' : '0',
'levtype' : 'pl',
'type' : 'an',
'class' : 'ei',
'param' : '132',
'area' : '31.925/77.575/31.825/77.675',
'levelist': '500/650/775/850/925/1000',
'target' : 'vpl.grb'
})
#
# (C) Copyright 2012-2013 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.
#
from ecmwfapi import ECMWFDataServer
# To run this example, you need an API key
# available from https://api.ecmwf.int/v1/key/
server = ECMWFDataServer()
server.retrieve({
'stream' : "oper",
'levtype' : "sfc",
'param' : "165.128/166.128/167.128",
'dataset' : "interim",
'step' : "0",
'grid' : "0.75/0.75",
'time' : "00/06/12/18",
'date' : "2014-07-01/to/2014-07-31",
'type' : "an",
'class' : "ei",
'format' : "netcdf",
'target' : "interim_2014-07-01to2014-07-31_00061218.nc"
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "1979-01-01/to/2017-12-31",
"expver": "1",
"grid": "0.75/0.75",
"area": "50/0/-50/180",
"levelist": "100/150/200/250/350/450/550/650/750/800/850/900/950/1000",
"levtype": "pl",
"param": "60.128",
"step": "0",
"stream": "oper",
"time": "00:00:00",
"type": "an",
"format": "netcdf",
"target": "EI_AN0012_1979_2017_PV60128_3D.nc",
})
def __init__(self):
"""
Initialises all variables needed. Sets the environment for the current
platform. Handy to have when testing on local platform.
"""
# This is one of the more important variables. This setting defines how far back into history (in days) we go.
# I.e. to do a whole year we would set this to 365 etc.
self.BACKFILL_DAYS = 10
self.home = expanduser("~")
self.server = ECMWFDataServer()
self.ERROR_FILES = []
self.DATE = datetime.datetime.today().strftime('%Y-%m-%d')
if sys.platform == "linux2":
self.MOTU_CLIENT_DIR = "/home/ubuntu/eumetsat_archive/motu-client-python"
self.FTP_DIR = "/home/ubuntu/mnt_s3_bucket"
self.DATA_FILE = "/home/ubuntu/eumetsat_archive/data.json"
self.OUTPUT_DIR = "/home/ubuntu/eumetsat_archive/temp"
self.COLOUR_FOLDER = "/home/ubuntu/eumetsat_archive/colour_files/"
else:
self.OUTPUT_DIR = self.home + "/repos/eumetsat_archive/temp"
self.DATA_FILE = "data.json"
self.FTP_DIR = self.home + "/repos/eumetsat_archive/ftpdir"
self.MOTU_CLIENT_DIR = self.home + "/repos/eumetsat_archive/motu-client-python"
self.COLOUR_FOLDER = self.home + "/repos/eumetsat_archive/colour_files/"
if not os.path.exists(self.OUTPUT_DIR):
os.makedirs(self.OUTPUT_DIR)
# Set up logging.
self.logFormatter = logging.Formatter(
"%(asctime)s [%(levelname)-5.5s] %(message)s")
self.logger = logging.getLogger()
self.handler = logging.StreamHandler()
self.formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
self.handler.setFormatter(self.formatter)
self.logger.addHandler(self.handler)
self.logger.setLevel(logging.INFO)
self.logPath = self.FTP_DIR
self.fileName = datetime.datetime.today().strftime('%Y-%m-%d')
if not os.path.exists(self.logPath + "/logs"):
os.makedirs(self.logPath + "/logs")
self.fileHandler = logging.FileHandler("{0}/{1}.txt".format(
self.logPath + "/logs", self.fileName))
self.fileHandler.setFormatter(self.logFormatter)
self.logger.addHandler(self.fileHandler)
self.fromdate = ""
self.todate = ""
self.datasetname = ""
argscounter = 0
self.counter = 0
self.dataCount = 0
# Get arguments. This enabled the scripts to be used via the following command;
# python batchDownload.py -fromdate 20161010 -todate 20161010 -datasetname CERSAT-GLO-BLENDED_WIND_L4-V3-OBS_FULL_TIME_SERIE
# Using -v will enable verbose logging, as well as maintaining the downloaded temp files.
for arg in sys.argv:
if arg == '-v':
self.debug = True
self.logger.setLevel(logging.DEBUG)
elif arg == '-fromdate':
self.fromdate = sys.argv[argscounter + 1]
elif arg == '-todate':
self.todate = sys.argv[argscounter + 1]
elif arg == '-datasetname':
self.datasetname = sys.argv[argscounter + 1]
elif arg == '-backfilldays':
self.BACKFILL_DAYS = int(sys.argv[argscounter + 1])
else:
self.debug = False
self.logger.setLevel(logging.INFO)
argscounter = argscounter + 1
pass
month_end = var_month_end
# Area
north = var_north
south = var_south
west = var_west
east = var_east
# Output folder, i.e. replace by your project name
out_path = 'var_out'
##################################################
#os.mkdir(out_path)
#os.mkdir(out_path+"nc_output_files")
#os.mkdir(out_path+"txt_output_files")
server = ECMWFDataServer()
count = 1
for year in range(year_start, year_end+1):
if len(range(year_start, year_end+1)) == 1:
mt_start = month_start
mt_end = month_end
else:
if year == year_start:
mt_start = month_start
mt_end = 12
elif year == year_end:
mt_start = 1
mt_end = month_end
else:
mt_start = 1
mt_end = 12
# conda activate ERA5
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class":
"ea",
"dataset":
"era5",
"expver":
"1",
"stream":
"oper",
"type":
"an",
"levtype":
"sfc",
"param":
"165.128/166.128/151.128",
"date":
"2015-09-29/to/2015-10-03",
"time":
"00/01/02/03/04/05/06/07/08/09/10/11/12/13/14/15/16/17/18/19/20/21/22/23",
"step":
"0",
"grid":
"0.25/0.25",
"area":
"30.5/-81.5/18/-65.5",
"format":
"netcdf",
class Downloader:
def __init__(self):
"""
Initialises all variables needed. Sets the environment for the current
platform. Handy to have when testing on local platform.
"""
# This is one of the more important variables. This setting defines how far back into history (in days) we go.
# I.e. to do a whole year we would set this to 365 etc.
self.BACKFILL_DAYS = 10
self.home = expanduser("~")
self.server = ECMWFDataServer()
self.ERROR_FILES = []
self.DATE = datetime.datetime.today().strftime('%Y-%m-%d')
if sys.platform == "linux2":
self.MOTU_CLIENT_DIR = "/home/ubuntu/eumetsat_archive/motu-client-python"
self.FTP_DIR = "/home/ubuntu/mnt_s3_bucket"
self.DATA_FILE = "/home/ubuntu/eumetsat_archive/data.json"
self.OUTPUT_DIR = "/home/ubuntu/eumetsat_archive/temp"
self.COLOUR_FOLDER = "/home/ubuntu/eumetsat_archive/colour_files/"
else:
self.OUTPUT_DIR = self.home + "/repos/eumetsat_archive/temp"
self.DATA_FILE = "data.json"
self.FTP_DIR = self.home + "/repos/eumetsat_archive/ftpdir"
self.MOTU_CLIENT_DIR = self.home + "/repos/eumetsat_archive/motu-client-python"
self.COLOUR_FOLDER = self.home + "/repos/eumetsat_archive/colour_files/"
if not os.path.exists(self.OUTPUT_DIR):
os.makedirs(self.OUTPUT_DIR)
# Set up logging.
self.logFormatter = logging.Formatter(
"%(asctime)s [%(levelname)-5.5s] %(message)s")
self.logger = logging.getLogger()
self.handler = logging.StreamHandler()
self.formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
self.handler.setFormatter(self.formatter)
self.logger.addHandler(self.handler)
self.logger.setLevel(logging.INFO)
self.logPath = self.FTP_DIR
self.fileName = datetime.datetime.today().strftime('%Y-%m-%d')
if not os.path.exists(self.logPath + "/logs"):
os.makedirs(self.logPath + "/logs")
self.fileHandler = logging.FileHandler("{0}/{1}.txt".format(
self.logPath + "/logs", self.fileName))
self.fileHandler.setFormatter(self.logFormatter)
self.logger.addHandler(self.fileHandler)
self.fromdate = ""
self.todate = ""
self.datasetname = ""
argscounter = 0
self.counter = 0
self.dataCount = 0
# Get arguments. This enabled the scripts to be used via the following command;
# python batchDownload.py -fromdate 20161010 -todate 20161010 -datasetname CERSAT-GLO-BLENDED_WIND_L4-V3-OBS_FULL_TIME_SERIE
# Using -v will enable verbose logging, as well as maintaining the downloaded temp files.
for arg in sys.argv:
if arg == '-v':
self.debug = True
self.logger.setLevel(logging.DEBUG)
elif arg == '-fromdate':
self.fromdate = sys.argv[argscounter + 1]
elif arg == '-todate':
self.todate = sys.argv[argscounter + 1]
elif arg == '-datasetname':
self.datasetname = sys.argv[argscounter + 1]
elif arg == '-backfilldays':
self.BACKFILL_DAYS = int(sys.argv[argscounter + 1])
else:
self.debug = False
self.logger.setLevel(logging.INFO)
argscounter = argscounter + 1
pass
def load_file(self):
"""
Loads the json file into memory.
"""
with open(self.DATA_FILE) as data_file:
self.jsondata = simplejson.load(data_file)
for item in self.jsondata:
if item["process"] == True:
self.dataCount = self.dataCount + 1
def run(self):
"""
Run function. This will decide if we have passed arguments or are running the process
completely.
"""
self.load_file()
if self.fromdate != "" and self.todate != "" and self.datasetname != "":
fromdate = datetime.datetime.strptime(self.fromdate,
"%Y%m%d").date()
toDate = datetime.datetime.strptime(self.todate, "%Y%m%d").date()
self.DATE = fromdate.strftime('%Y-%m-%d')
item = self.findItem(self.datasetname)
self.logger.info("==== Processing " + self.datasetname.upper() +
" =====")
##TODO: In here at some point let's get the proper dates to enable using ranges.
##i.e. transofrm fromdate / todate to actual dates maybe?
self.processItem(item, fromdate, toDate)
else:
self.processJsonData()
def findItem(self, datasetname):
"""
For the selected dataset via the command line, get the object from the JSON file
"""
for item in self.jsondata:
if item["dataset"] == datasetname:
##TODO: Enable the use of variables.
return item
def processItem(self, item, fromdate, toDate):
""" This is the main part of the program. Here we retrieve the parameters from the
object that we get from the JSON file.
:param item: object from the JSON file.
:param fromdate: from which date to start processing
:param toDate: what the end date for processing should be
"""
dataset = item["dataset"]
try:
variable = item["variable"]
except:
pass
try:
te = item["te"]
except:
te = " -180 -90 180 90"
variableName = ""
try:
variableName = "_" + item["variableName"]
except:
variableName = ""
try:
z = item["depth"]
except:
z = -99
try:
ts = item["ts"]
except:
ts = "8192 0"
dateformat = item["dateformat"]
ftpdatafolder = self.FTP_DIR + "/" + dataset.upper(
) + variableName.upper()
file = ftpdatafolder + "/" + fromdate.strftime('%Y-%m-%d') + ".png"
# If the file exists - don't do anything.
if not os.path.exists(file):
self.logger.info(
fromdate.strftime('%Y-%m-%d') +
".png doesn't exist. Backfilling...")
fromdate = fromdate.strftime(dateformat)
toDate = toDate.strftime(dateformat)
if item["type"] == 'useMotu':
# If this is a dataset coming from the MOTU download service, handle it accordingly.
username = item["username"]
password = item["password"]
m = item["m"]
s = item["s"]
self.getMotuDataset(username, password, m, s, dataset,
fromdate, toDate, variable, te, z,
variableName, ts)
elif item["type"] == 'ecmwf':
# If this uses the ECMWF python download service, handle it accordingly.
self.logger.info("Processing ECMWF data")
ecmfwdataset = item["ecmwf_dataset"]
try:
time = item["time"]
except:
time = "12:00:00"
self.getECMWFDataset(dataset, ecmfwdataset, fromdate,
item["param"], "", ts, item["expver"],
time)
elif item["type"] == 'ftp':
# Otherwise it's an FTP download.
url = item["url"]
self.getFTPDataset(url, dataset, variable, fromdate, te, ts)
else:
self.logger.info("Already got " + fromdate.strftime('%Y-%m-%d') +
".png Moving on.")
def cropData(self, dataset):
"""
For the MACC datasets, they all need to be cropped to a certain size, this is because there is a column
of data on the far right which contains header information, and renders as null data.
"""
self.logger.info("Cropping")
ftpdatafolder = dataset.upper()
filename = str(self.FTP_DIR) + '/' + str(ftpdatafolder) + '/' + str(
self.DATE) + '.png'
cropCommand = "convert " + filename + " -crop 8183x4096+0+0 +repage + " + filename
subprocess.call(cropCommand, shell=True)
def processJsonData(self):
"""
If we are processing the entire data.json file then this is where we do it.
"""
self.logger.info(" Found " + str(self.dataCount) +
" datasets to process. Starting process.")
for item in self.jsondata:
# If we want to process the data according to the 'process' flag on the object.
if item["process"] == True:
self.counter = self.counter + 1
dataset = item["dataset"]
self.logger.info("=========== " + str(self.counter) + " / " +
str(self.dataCount) + " : " + dataset +
" =============")
try:
gap = item["gap"]
except:
gap = 0
variableName = ""
try:
variableName = "_" + item["variableName"]
except:
variableName = ""
daysago = item["daysAgo"]
self.logger.info("==== Processing " + dataset.upper() +
variableName + " =====")
for num in range(daysago, self.BACKFILL_DAYS + 1):
now = datetime.datetime.today()
numSubtractionDays = datetime.timedelta(days=num)
fromdate = now - numSubtractionDays
toDate = fromdate + datetime.timedelta(days=gap)
self.DATE = fromdate.strftime('%Y-%m-%d')
self.processItem(item, fromdate, toDate)
if len(self.ERROR_FILES) > 0:
self.logger.error(str(len(self.ERROR_FILES)) + " file(s) failed.:")
for file in self.ERROR_FILES:
self.logger.info(file)
else:
self.logger.info("All successful.")
self.cleanup()
self.logger.info("=====================================")
self.logger.info("=====================================")
def getMotuDataset(self, username, password, m, s, dataset, fromdate,
todate, variable, te, depth, variableName, ts):
"""
Function to retrieve datasets that are supported by the MOTU python client.
Parameters are fairly self explanatory.
"""
filename = dataset + variableName + "_" + self.DATE
tempfilename = self.OUTPUT_DIR + "/" + filename + ".nc"
if os.path.exists(tempfilename):
self.logger.info(tempfilename +
" already exists so just reprocessing it.")
else:
self.logger.info("Downloading " + filename)
if self.debug:
motucommand = "python " + self.MOTU_CLIENT_DIR + "/motu-client.py "
else:
motucommand = "python " + self.MOTU_CLIENT_DIR + "/motu-client.py -q "
motucommand = (motucommand + " -u " + username + " -p " +
password + " -m " + m + " -s " + s + " -d " +
dataset + " -x -180 -X 180 -y -90 -Y 90 " + " -t " +
fromdate + " -T " + todate + " -v " + variable +
" -o " + self.OUTPUT_DIR + " -f " + filename +
".nc")
# Occasionally datasets are requested at specific depths. If this is the case
# we create a range for the depth variable.
if depth != -99:
motucommand = (motucommand + " -z " + str(depth) + " -Z " +
str(depth + 0.0002))
self.logger.debug(motucommand)
dataset = dataset + variableName
subprocess.check_output(motucommand, shell=True)
try:
self.processImagery(dataset, filename, variable, te, self.DATE, ts)
self.moveToFtpFolder(dataset, filename + ".png", fromdate)
self.cleanup()
except:
self.logger.error("Error processing " + filename)
self.ERROR_FILES.append(filename)
pass
def getECMWFDataset(self, dataset, ecmfwdataset, fromdate, param, te, ts,
expver, time):
"""
When downloading ECMWF data we use the ECMWF python api. This allows us to specify
various parameters, the most important one the "param" one - which indicates the
dataset that we're after. the "grid" specifies the resolution (in pixels per km2).
"""
self.logger.info("Processing " + dataset)
filename = dataset + "_" + self.DATE
tempfilename = self.OUTPUT_DIR + "/" + filename + ".nc"
if os.path.exists(tempfilename):
self.logger.info(tempfilename +
" already exists so just reprocessing it.")
else:
ecmwfCommand = 'server.retrieve({' \
'"class": "mc", ' \
'"dataset": "' + ecmfwdataset + '",' \
'"date": "' + fromdate + '",' \
'"expver": "' + expver + '",' \
'"grid": "0.1/0.1",' \
'"levtype": "sfc",' \
'"param": "' + param + '",' \
'"step": "3",' \
'"stream": "oper",' \
'"time": "' + time + '",' \
'"type": "fc",' \
'"format": "netcdf",' \
'"target": "' + self.OUTPUT_DIR + "/" + filename + ".nc" + '"})'
self.logger.debug(ecmwfCommand)
self.server.retrieve({
"class":
"mc",
"dataset":
ecmfwdataset,
"date":
fromdate,
"expver":
expver,
"grid":
"0.1/0.1",
"levtype":
"sfc",
"param":
param,
"step":
"3",
"stream":
"oper",
"time":
time,
"type":
"fc",
"format":
"netcdf",
"target":
self.OUTPUT_DIR + "/" + filename + ".nc"
})
self.processImagery(dataset, filename, "", " 0 -90 360 90", self.DATE,
ts)
self.moveToFtpFolder(dataset, filename + ".png", fromdate)
self.cropData(dataset)
self.cleanup()
def getFTPDataset(self, url, dataset, variable, fromdate, te, ts):
"""
This will download data from most FTP sites. The file construction just needs to be created properly.
"""
filename = dataset + "_" + fromdate
formattedFileName = dataset + "_" + self.DATE
tempfilename = self.OUTPUT_DIR + "/" + formattedFileName + ".nc"
if os.path.exists(tempfilename):
self.logger.info(tempfilename +
" already exists so just reprocessing it.")
else:
self.logger.debug("Downloading " + filename)
if self.debug:
wgetCommand = ("wget " + url + filename + ".nc -O " +
self.OUTPUT_DIR + "/" + formattedFileName +
".nc")
self.logger.info(wgetCommand)
else:
wgetCommand = ("wget -q " + url + filename + ".nc -O " +
self.OUTPUT_DIR + "/" + formattedFileName +
".nc")
subprocess.call(wgetCommand, shell=True)
self.processImagery(dataset, formattedFileName, variable, te, fromdate,
ts)
self.moveToFtpFolder(dataset, formattedFileName + ".png", fromdate)
self.cleanup()
def processImagery(self, dataset, filename, myvariable, te, fromdate, ts):
"""
Once the netcdf data is downloaded, this function will convert the imagery to a tiff file
and then convert that tiff file to a png with the correct parameters.
"""
if os.path.exists(self.OUTPUT_DIR + "/" + filename + ".nc"):
if myvariable:
myvariable = ":" + myvariable
else:
myvariable = ""
self.logger.info("Processing " + filename)
if self.debug:
gdalwarpcommand = "gdalwarp "
else:
gdalwarpcommand = "gdalwarp -q "
gdalwarpcommand = gdalwarpcommand + " \
-of GTiff \
-t_srs epsg:4326 \
-te" + te + " \
-ts " + ts + " \
NETCDF:" + self.OUTPUT_DIR + "/" + filename + ".nc" + myvariable + " " + \
self.OUTPUT_DIR + "/" + filename + ".tif"
if self.debug:
gdaldemcommand = "gdaldem color-relief "
else:
gdaldemcommand = "gdaldem color-relief -q "
gdaldemcommand = gdaldemcommand + " \
-of PNG -alpha " + \
self.OUTPUT_DIR + "/" + filename + ".tif " + \
self.COLOUR_FOLDER + dataset + ".txt " + \
self.OUTPUT_DIR + "/" + filename + ".png"
self.logger.debug(gdalwarpcommand)
self.logger.debug(gdaldemcommand)
subprocess.call(gdalwarpcommand, shell=True)
subprocess.call(gdaldemcommand, shell=True)
else:
self.logger.info(filename + " doesn't exist. Moving on.")
def moveToFtpFolder(self, target_foldername, filename, fromdate):
"""
After processing, the png file is currently in a temporary folder. This function
moves the file to the correct folder according to the global variable FTP_DIR
"""
self.logger.info("Moving")
ftpdatafolder = target_foldername.upper()
if not os.path.exists(self.FTP_DIR + '/' + ftpdatafolder):
os.makedirs(self.FTP_DIR + '/' + ftpdatafolder)
origin = str(self.OUTPUT_DIR) + '/' + str(filename)
destination = str(self.FTP_DIR) + '/' + str(ftpdatafolder) + '/' + str(
self.DATE) + '.png'
self.logger.debug('from: ' + origin)
self.logger.debug('to: ' + destination)
subprocess.call("mv " + origin + " " + destination, shell=True)
subprocess.call("chmod 644 " + destination, shell=True)
self.logger.info("Processing finished")
try:
self.logger.debug("Removing: " + self.OUTPUT_DIR + "/" +
target_foldername + "_" + fromdate + ".png")
os.remove(self.OUTPUT_DIR + "/" + target_foldername + "_" +
fromdate + ".png")
except:
pass
def cleanup(self):
"""
This function removes all temporary files, unless we're in debug mode.
"""
if self.debug:
pass
else:
try:
filelist = os.listdir(self.OUTPUT_DIR)
for f in filelist:
os.remove(self.OUTPUT_DIR + "/" + f)
self.logger.info("Temp files deleted")
except:
self.logger.info("Temp files not deleted")
pass
from ecmwfapi import ECMWFDataServer
if __name__ == '__main__':
server = ECMWFDataServer()
server.retrieve({
"class": "ti",
"dataset": "tigge",
"date": "2012-01-01/to/2012-12-31",
"expver": "prod",
"grid": "0.5/0.5",
"levtype": "sfc",
"origin": "ecmf",
"param": "59/151/165/166/167",
"area": "-33/135/-41/150",
"step": "0/6/12/18/24/30/36/42/48/54/60/66/72",
"time": "00:00:00",
"type": "fc",
"target": "2012fc",
})
server.retrieve({
"class": "ti",
"dataset": "tigge",
"date": "2012-01-01/to/2012-12-31",
"expver": "prod",
"grid": "0.5/0.5",
"levtype": "sfc",
"origin": "ecmf",
"param": "59/151/165/166/167",
"area": "-33/135/-41/150",
#!/usr/bin/env python
# Use this script to download ERA Interim data needed to compute hydrological (not atmosphe)
# Make sure the home folder contains .ecmwfapirc, see https://software.ecmwf.int/wiki/display/WEBAPI/Access+ECMWF+Public+Datasets
# To install 'ecmwfapi': sudo pip install https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
# Set year/month/day to be downloaded. This will save all data into one file for conversion.
# Thus, do not request too long interval!
start_year = '2016'
start_month = '01-01'
end_year = '2016'
stop_month = '12-31'
# Set time to be downloaded (usually 12:00:00)
# Possible values: '00:00:00/06:00:00/12:00:00/18:00:00'
time = '12:00:00';
# Set output file.
output_file = 'i:/GlobalModel/ERAinterim/Surface/OriginalForHydro/ERA_SM_and_Snow.nc'
# prepare/declare arguments for downloading (will be modified later)
args_sl = {
"class": "ei",
"dataset": "interim",
"date": "2015-02-01/to/2016-12-31",
"expver": "1",
"grid": "0.75/0.75",
"levtype": "sfc",
"param": "39.128/40.128/41.128/42.128/141.128",
"step": "0",
#!/usr/bin/env python
# Surface temp 167.128
# Pressure 54.128 levtype =
# Relative humidity 157.128
# specific humidity .128
# total cloud cover .128
from ecmwfapi import ECMWFDataServer
#periode = "1979-01-01/to/2018-07-31"
periode = "1990-12-01/to/1990-12-08"
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"expver": "1",
"stream": "oper",
"type": "an", # use fc for forcast data
"levtype": "sfc", #surface variable
"param": "164.128",
"date": periode,
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
#"step": "0", # timestep is 3 hours
"grid": "0.75/ 0.75", # 0.75, 0.75 is recomended resolution for grib, 1.0/1.0 is recomended for when you dont do global.
"area": "75/-15/30/42", # "75/-15/30/42", #Europa --> N/W/S/E lat long degrees
"format":"netcdf",
"target": "./filesML/total_cloud_cover_Europa_sp.nc"
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
#list_dates <- seq.Date(as.Date(data_i), as.Date(data_f), by = "day")
#list_dates <- format(list_dates, "%Y%m%d")
server.retrieve({
#'stream' : "oper",
'levtype' : "sfc",
'param' : "172/134/151/165/166/167/168/169/235/33/34/31/141/139/170/183/236/39/40/41/42",
'dataset' : "interim",
'step' : "0",
'grid' : "128",
'time' : "00/06/12/18",
'date' : "20100701/to/20100701",
'type' : "an",
'class' : "ei",
#'format' : "netcdf",
'target' : "ERA-Int_sfc_20100701.grib"
})
code = {'u10': '165.128', 'v10': '166.128'}
time = "2008-01-01_2018-01-01"
download_name = 'uv_{}.nc'.format(time)
data_path = "/home/qxs/bma/"
download_file = data_path + download_name
param = ''
vars = ['u10', 'v10']
for var in vars:
param = param + str(code[var]) + '/'
try:
os.makedirs(data_path + time + '/')
except OSError:
pass
finally:
if not os.path.exists(download_file):
server = ECMWFDataServer()
server.retrieve({
'class': "ei",
'dataset': "interim",
'stream': "oper",
'levtype': "sfc",
'type': "an",
'time': "00/12",
'step': "0",
'grid': "0.5/0.5",
'area': area["china"],
'param': param[:-1],
'date': "{}/to/{}".format("20080101", "20180101"),
'format': "netcdf",
'target': download_file
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
'class': "ei",
'dataset': "interim",
'type': "fc",
'stream': "oper",
'expver': "1",
'levtype': "sfc",
'param': "228.128",
'step': "3",
'grid': "0.75/0.75",
'time': "00:00:00/12:00:00",
'date': "1979-01-01/to/2017-12-31",
'format': "netcdf",
'target': "EI_1979_2017_TP228128.nc"
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
import sys
str = sys.argv[1]
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": str,
"expver": "1",
"levelist": "46/to/60",
"levtype": "ml",
"param": "130.128/131.128/132.128/138.128",
"step": "0",
"stream": "oper",
"target": "2015/" + str.replace('-', '') + "_ml.nc",
"format": "netcdf",
"time": "00/06/12/18",
"type": "an",
"area": "75/-20/30/70",
"grid": "0.36/0.36",
})
lonmin = str(float(lonmin)-dl)
lonmax = str(float(lonmax)+dl)
latmin = str(float(latmin)-dl)
latmax = str(float(latmax)+dl)
print 'lonmin-dl = ',lonmin
print 'lonmax+dl =',lonmax
print 'latmin-dl =',latmin
print 'latmax+dl =',latmax
######################
# Retrieve data
######################
server = ECMWFDataServer()
### Loop on the years and months
for year in range(yearStart, yearEnd+1):
if year==yearStart and year==yearEnd:
monthStartYear = monthStart
monthEndYear = monthEnd
elif year==yearStart and year!=yearEnd:
monthStartYear = monthStart
monthEndYear = 12
elif year!=yearStart and year==yearEnd:
monthStartYear = 1
monthEndYear = monthEnd
else:
monthStartYear = 1
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "2013-01-01/to/2013-01-31",
"expver": "1",
"grid": "0.125/0.125",
"levelist":
"400/450/500/550/600/650/700/750/775/800/825/850/875/900/925/950/975/1000",
"levtype": "pl",
"param": "130.128",
"step": "0",
"stream": "oper",
"area": "-2/37/-4/38",
"target": "2013001_2013031__temperature.nc",
"time": "12",
"type": "an",
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
import sys
str=sys.argv[1]
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": str,
"expver": "1",
"levelist": "46/to/60",
"levtype": "sfc",
"param": "134.128/167.128",
"step": "0",
"stream": "oper",
"target": "2013/"+str.replace('-','')+"_sfc.nc",
"format": "netcdf",
"time": "00/06/12/18",
"type": "an",
"area": "60/70/10/150",
"grid": "0.36/0.36",
})
from ecmwfapi import ECMWFDataServer
from pathlib import Path
from uclgeog.geog_data import procure_dataset
ecmwf_file = 'europe_data_2016_2017.nc'
# pull the years info from ifile
# if the file is multiple years eg europe_data_2010_2011.nc
# then split it into multiple files
years = np.array(Path(ecmwf_file).stem.split('_'))[2:].astype(int)
if not (Path('data') / ecmwf_file).exists():
# try to get it from UCL servers
done = procure_dataset(ofile, verbose=True)
if not done:
server = ECMWFDataServer()
print('requesting data ... may take some time')
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": f"{years[0]}-01-01/to/{years[1]+1}-01-01", # Time period
"expver": "1",
"levtype": "sfc",
"param":
"2t", # Parameters. Here we use 2m Temperature (2t) See the ECMWF parameter database, at http://apps.ecmwf.int/codes/grib/param-db
"stream": "oper",
"type": "an",
"time": "00/60/12/18",
"step": "0",
"area":
"75/-20/10/60", # Subset or clip to an area, here to Europe. Specify as North/West/South/East in Geographic lat/long degrees. Southern latitudes and Western longitudes must be given as negative numbers.
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 19 12:20:00 2020
ERA-Interim api request template
@author: Michael Tadesse
"""
import os
os.chdir('D:\\data\\era_interim\\era_interim_netcdf')
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": '2007-01-01/to/2007-12-31',
"expver": "1",
"grid": "0.75/0.75",
"levtype": "sfc",
"param": '34.128',
"step": "0",
"stream": "oper",
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
"type": "an",
"format": "netcdf",
"target": 'era_interim_sst_2007_.nc',
})
string=""
range_dates=""
range_hdates=""
range_end=eval(string.join(["len(starts_", month_name_list[dm-1] , ")"]))
for kd in range(1,range_end):
range_hdates += string.join([string.join([str(dy),eval(string.join(["starts_" , month_name_list[dm-1] , "[kd-1]"]))]),"/"])
range_dates += string.join([string.join([str(2017),eval(string.join(["starts_", month_name_list[dm-1] , "[kd-1]"]))]),"/"])
continue
# adds final date without the final /
range_hdates += string.join([str(dy),eval(string.join(["starts_" , month_name_list[dm-1] , "[range_end-1]"]))])
range_dates += string.join([str(2017),eval(string.join(["starts_",month_name_list[dm-1] , "[range_end-1]"]))])
file_name=[ pathto, "ECMWF_", variable, "_", region, "_pf_reforecast_", str(dy),"_", month_name_list[dm-1] , ".grib"];
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "s2",
"dataset": "s2s",
"hdate": range_hdates,
"date": range_dates,
"origin": "ecmf",
"expver": "prod",
"levtype": level_type,
"level":level,
"param": "167",
"step": paso,
"stream": "enfh",
"target": string.join(file_name),
"area": region_limits,
"time": "00",
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
# get all of the Nov-Mar data
for YEAR in range(1996, 2017):
for MONTH in [11, 12, 1, 2, 3]:
qmonth = str(MONTH).zfill(
2
) # in some months 9 starts are available, i am only using the first 8 for each month.
if MONTH == 1:
day_list = [4, 7, 11, 14, 18, 21, 25, 28]
if MONTH == 2:
day_list = [1, 4, 8, 11, 15, 18, 22, 25] #,29]
if MONTH == 3:
day_list = [3, 7, 10, 14, 17, 21, 24, 28] #,31]
if MONTH == 11:
day_list = [3, 7, 10, 14, 17, 21, 24, 28]
if MONTH == 12:
day_list = [1, 5, 8, 12, 15, 19, 22, 26] # ,29]
for DAY in day_list: # these are the days the fcast starts at.
server.retrieve({
"class":
"s2",
"dataset":
"s2s",
"date":
"2016-" + str(MONTH).zfill(2) + "-" + str(DAY).zfill(2),
"expver":
"prod",
#!/usr/bin/env python
# eastward vapour flux
# northward vapour flux
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
for d in [2015, 2016, 2017]:
# print("{y}-06-01/to/{y}-09-30".format(y=d))
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "{y}-06-01/to/{y}-12-31".format(y=d),
"expver": "1",
"grid": "0.25/0.25",
"levtype": "sfc",
"param": "71.162/72.162",
"step": "0",
# "3,6,9,12",
"stream": "oper",
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
# "time": "00:00:00/12:00:00",
"area": "11/70/5/76",
"type": "an",
"target": "mf_{y}.grib".format(y=d),
# "format": "netcdf",
})
def main(argv):
try:
opts,argv = getopt.getopt(argv,":h:i:e:s:E:o:g:P:t:f:r:",['help','[outFile]','code','[shapeFile]','start','end','[tr]'])
except getopt.GetoptError:
print 'error in parameter for eraInterimDownload. type eraInterimDownload.py -help for more detail on use '
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'eraInterimDownload.py '
print ' [mandatory] : '
print ' --init <dateStart YYYY-MM-DD>'
print ' --end <dateEnd YY-MM-DD>'
print ' --shapefile <shapefile> OU -Extend < xmin,ymax,xmax,ymin>'
print ' [optional] :'
print ' --typeData < analyse , forcast> (default forcast)'
print ' --grid <EraInterim Time> (default 0.75)'
print ' --outfile <outfolder> (default /home/user/eraInterim)'
print ' --proxy <True/False> (default False)'
print ' --temporaryFile <True/False> (default False)'
print ' --result < TxtFile / RasterFile> default RasterFile'
print ''
sys.exit()
elif opt in ('-o','--outFolder'):
oFolder = arg
elif opt in ('-i','--start'):
startDate = arg
elif opt in ('-e','--end'):
endDate = arg
elif opt in ('-s','--shapefile'):
pathToShapefile = arg
elif opt in ('-E','--tr'):
extend = arg.split(',')
elif opt in ('-g','--grid'):
grid = arg
elif opt in ('-P','--proxy'):
proxy = arg
elif opt in ('-t','--typeData'):
typeData = arg
elif opt in ('-f','--temporaryFile'):
temporaryFile = arg
elif opt in ('-r','--result'):
typeOutput = arg
if len(sys.argv) < 7:
print 'eraInterimDownload.py'
print ' -i <dateStart YYYY-MM-DD> '
print ' -e <dateEnd YY-MM-DD>'
print ' -s <shapefile> '
print ' or'
print ' -E < xmin,ymax,xmax,ymin>]'
print ''
print ' [-t < analyse , forcast> (default analyse)]'
print ' [-g <size of grid in 0.125/0.25/0.5/0.75/1.125/1.5/2/2.5/3> (default0.75)]'
print ' [-o <outfolder> (default /home/user/eraInterim)]'
print ' [-P <proxy : True/False> (default False)]'
print ' [-f <temporaryFile : True/False> (default False)]'
print ' [-r <resultOutput : TxtFile/RasterFile> (default RasterFile)]'
print ''
print 'For help on interimCode -help'
sys.exit(2)
try:
oFolder
except NameError:
oFolder = os.path.expanduser('~')
oFolder = oFolder + '/eraInterim'
print "output folder not precised : downloaded eraInterim images on "+oFolder
# verification du folder/or creation if not exists
utils.checkForFolder(oFolder)
try:
startDate
except NameError:
exit ('init Date not precised')
# verification si sartDate est une date
startDate=utils.checkForDate(startDate)
try:
endDate
except NameError:
exit ('end Date not specified')
# verification si sartDate est une date
endDate=utils.checkForDate(endDate)
if (startDate>endDate):
exit('startDate could not be greater than endDate')
today=date.today()
limitDate = today - timedelta(days=31*3)
limitDate=date(limitDate.year,limitDate.month,calendar.monthrange(limitDate.year,limitDate.month)[1])
if (startDate>limitDate or endDate>limitDate ):
exit('date could not exceed 2014-12-31')
try:
pathToShapefile
except NameError:
try:
extend
except NameError:
exit ('no Area of interest have been specified. please use -shp or -tr to declare it')
if 'pathToShapefile' in locals():
extendArea=utils.convertShpToExtend(pathToShapefile)
else:
extendArea=extend
extendArea=utils.checkForExtendValidity(extendArea)
try:
typeData
except NameError:
typeData='analyse'
try:
grid
except NameError:
grid='0.75'
grid=utils.checkForGridValidity(grid)
try:
proxy
except NameError:
proxy=False
#Proxy parameteres needed
if(proxy):
login = raw_input('login proxy : ')
pwd = raw_input('password proxy : : ')
site = raw_input('site (surf.cnes.fr) : ')
os.environ["http_proxy"] = "http://%s:%s@%s:8050"%(login,pwd,site)
os.environ["https_proxy"] = "http://%s:%s@%s:8050"%(login,pwd,site)
try:
temporaryFile
except NameError:
temporaryFile=False
try:
typeOutput
except NameError:
typeOutput='RasterFile'
"""----------------------------------------"""
#Create param if first Time
if (not utils.checkForFile(os.path.expanduser('~')+'/.ecmwfapirc')):
print ('for first connexion you have to define yout key and password on ecmwf')
print ('cf https://apps.ecmwf.int/auth/login/')
print ('')
u = raw_input('user (mail) : ')
k = raw_input('keys : ')
utils.createParamFile(os.path.expanduser('~')+'/.ecmwfapirc',u,k)
delta = endDate - startDate
nbDays = delta.days + float(delta.seconds) / 86400 + 1
#--------------------------On charge les rasters
if typeData == "analyse":
time = ['00',"12","06","18"]
step = []
nbBandByDay=len(time)
else:
time = ['00',"12"]
step = [3,6,9,12]
nbBandByDay=(12*len(time))/(len(step))+1
server = ECMWFDataServer()
""" altitude de la grille EraInterim """
# Only Forcast possible
codeGeopot= [129]
GeoFile = oFolder+"/129"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeGeopot, GeoFile,typeData)
server.retrieve(struct)
Geo = utils.convertNETCDFtoDicArray(GeoFile)
Geo = utils.convertGeoToAlt(Geo)
#un peu inutile car ne change pas ... mais bon!
Geo = utils.computeDailyMax(Geo, nbBandByDay, typeData)
""" Vitesse du vent """
codeVent= [165,166]
vent={}
ventFile=[]
for i in codeVent:
ventFile.append(oFolder+"/"+str(i)+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc')
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, [i], ventFile[-1],typeData)
server.retrieve(struct)
vent[i]=utils.convertNETCDFtoDicArray(ventFile[-1])
vent = utils.fusVentFromDict(vent,nbBandByDay)
vent=utils.computeDailyMean(vent,nbBandByDay,typeData)
""" Humidité relative """
codePressure= [134]
pressureFile = oFolder+"/134"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codePressure, pressureFile,typeData)
server.retrieve(struct)
pressure = utils.convertNETCDFtoDicArray(pressureFile)
#oulalal c'est moche
pressureMean = utils.convertPaToKgPa(pressure)
pressure = utils.convertToHectoPascal(pressure)
pressureMean = utils.computeDailyMean(pressureMean,nbBandByDay,typeData)
codeT2m= [167]
T2mFile = oFolder+"/167"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeT2m, T2mFile,typeData)
server.retrieve(struct)
T2m = utils.convertNETCDFtoDicArray(T2mFile)
Tmean = utils.computeDailyMean(T2m, nbBandByDay, typeData)
Tmax = utils.computeDailyMax(T2m,nbBandByDay)
Tmin = utils.computeDailyMin(T2m,nbBandByDay)
T2m = utils.convertKToD(T2m)
#T2m = utils.computeDailyMean(T2m,nbBandByDay,typeData)
codeDewP= [168]
DewPFile = oFolder+"/168"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeDewP, DewPFile,typeData)
server.retrieve(struct)
DewP = utils.convertNETCDFtoDicArray(DewPFile)
DewP = utils.convertKToD(DewP)
#DewP = utils.computeDailyMean(DewP,nbBandByDay,typeData)
humidity = utils.ComputeHumidityFromPT(pressure,T2m,DewP)
#humidity = utils.computeDailyMean(humidity,nbBandByDay,typeData)
Hmax = utils.computeDailyMax(humidity,nbBandByDay)
Hmin = utils.computeDailyMin(humidity,nbBandByDay)
Hmean = utils.computeDailyMean(humidity,nbBandByDay,typeData)
""" ONLY FORCAST FOR THESE VAR"""
typeData="forcast"
time = ['00',"12"]
step = [3,6,9,12]
nbBandByDay=(12*len(time))/(len(step))+1
""" Rayonnement global incident journalier """
# Only Forcast possiblet
codeRay= [176]
RayFile = oFolder+"/176"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeRay, RayFile,typeData)
server.retrieve(struct)
Ray = utils.convertNETCDFtoDicArray(RayFile)
Ray = utils.computeDailyMean(Ray,nbBandByDay,typeData)
Ray = utils.convertWToMJ(Ray)
""" downward surface solar radiation """
# Only Forcast possiblet
codeRay= [169]
RayFileDownShort = oFolder+"/169"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeRay, RayFileDownShort,typeData)
server.retrieve(struct)
RayDownShort = utils.convertNETCDFtoDicArray(RayFileDownShort)
RayDownShort = utils.computeDailyMean(RayDownShort,nbBandByDay,typeData)
RayDownShort = utils.convertWToMJ(RayDownShort)
""" downward surface thermal radiation """
# Only Forcast possiblet
codeRay= [175]
RayFileDownLong = oFolder+"/175"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeRay, RayFileDownLong,typeData)
server.retrieve(struct)
RayDownLong = utils.convertNETCDFtoDicArray(RayFileDownLong)
RayDownLong = utils.computeDailyMean(RayDownLong,nbBandByDay,typeData)
RayDownLong = utils.convertWToMJ(RayDownLong)
""" Evaporation """
codeEvap= [182]
EvapFile = oFolder+"/182"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codeEvap, EvapFile,typeData)
server.retrieve(struct)
Evap = utils.convertNETCDFtoDicArray(EvapFile)
Evap = utils.computeDailyMean(Evap,nbBandByDay,typeData)
#Evap = utils.convertMToMm(Evap)
""" Precipitation """
#NOT NEEDED FOR ETO BUT Exported
utils.checkForTimeValidity(time)
utils.checkForStepValidity(step,typeData)
codePrecipitation= [228]
precipitationFile = oFolder+"/228"+'_'+startDate.strftime('%Y%m%d')+'_'+endDate.strftime('%Y%m%d')+'.nc'
struct=utils.create_request_sfc(startDate, endDate, time, step, grid, extendArea, codePrecipitation, precipitationFile)
server.retrieve(struct)
precipitation = utils.convertNETCDFtoDicArray(precipitationFile)
precipitation=utils.computeDailyAccumulation(precipitation,nbBandByDay,typeData)
""" Grid of latitude [0],longitude[1] in WGS84"""
geoTransform=utils.getGeoTransform(RayFile)
shape=Ray[0].shape
latlon = utils.getCentroidLatFromArray(shape,geoTransform,grid)
""" --------------------- Compute ET0---------------------- """
ET0_0={}
ET0_1={}
ET0_2={}
DoyList=[]
DateList=[]
for i in range(0,int(nbDays)):
#jour Julien
J = utils.doy(startDate,i)
dateEnCours=startDate+ timedelta(days=i)
DateList.append(dateEnCours)
DoyList.append(J)
Hmax[i] = np.where(Hmax[i]>100,100,Hmax[i])
# --- Constants ---#
#Solar constant
Gsc = 0.0820 # [MJ.m-2.min-1]
#Albedo - grass reference crop
a = 0.23
#Ratio of molecular weight of water vapor/dry air
epsilon=0.622
#Latente heat of vaporisation
Lv=2.45 # [MJ.kg-1]
# Specific heat at constant pressure
Cp= 1.013e-3; # [MJ.kg-1.°C-1]
# Stefan-Boltzmann constant [MJ.K-4.m-2.day-1]
StefBoltz=4.903e-9; #FAO
# --- Equations ---#
# Psychometric constant [kPa.°C-1]
cte_psy = (Cp*pressureMean[i])/(epsilon*Lv) # Equation 8 Chap 3 FAO
#Mean sturation vapor presure [kPa]
#es = (utils.esat(pressureMean[i],Tmax[i]) + utils.esat(pressureMean[i],Tmin[i]))/2; #Equation 12 Chap 3
es = (utils.eocalc(Tmax[i]-273.16)+utils.eocalc(Tmin[i]-273.16))/2 #Equation 12 Chap 3
# Slope of saturation vapour pressure curve at air temperature [kPa.°C-1]
delta = utils.delta_calc(Tmean[i]); # Equation 13 Chap 3
# Actual vapour pressure derived from relative humidity [kPa]
#ea = (utils.esat(pressureMean[i]/100,Tmax[i]-273.16)*(Hmax[i]/100) + utils.esat(pressureMean[i]/100,Tmin[i]-273.16)*(Hmin[i]/100))/2; # Equation 17 Chap 3
ea = (utils.eocalc(Tmax[i]-273.16)*(Hmax[i]/100)+utils.eocalc(Tmin[i]-273.16)*(Hmin[i]/100))/2
# Conversion of latitude from degrees to radians
phi = (np.pi/180)* latlon[1];
# Relative distance Earth-Sun
dr = 1 + 0.033*math.cos(2*math.pi*J/365); # Equation 23 Chap 3
# Solar declination
d = 0.4093*math.sin(2*math.pi*J/365 - 1.39); # Equation 24 Chap 3
# sunset hour angle
ws = np.arccos(-np.tan(phi)*math.tan(d)); # Equation 25 Chap 3
"""Classical calculation FAO """
# Extraterestrial radiation for daily periods
Ra = (24.*60/np.pi)*Gsc*dr*(ws*np.sin(phi)*np.sin(d) + np.cos(phi)*np.cos(d)*np.sin(ws)) # Equation 21 Chap 3
# Clear sky solar radiation [MJ.m-2.day-1]
Rso = (0.75 + 2e-5*Geo[i])*Ra; # Equation 37 Chap 3
# Net solar radiation [MJ.m-2.day-1]
Rns = (1 - a)*RayDownShort[i]; # Equation 38 Chap 3
#
f=(1.35*(np.fmin(RayDownShort[i]/Rso,1)) - 0.35);
# Net longwave radiation [MJ.m-2.day-1]
Rnl = StefBoltz*((Tmax[i]**4 + Tmin[i]**4)/2)*(0.34 - 0.14*np.sqrt(ea))*f; # Equation 39 Chap 3
# Net Radiation [MJ.m-2.day-1]
Rn =Rns - Rnl; # Equation 40 Chap 3
G = 0; # Equation 42 Chap 3
ET0_0[i] = ( 0.408*delta*( Rn-G )+ cte_psy*( 900/(Tmean[i] + 273) )*(es - ea)*vent[i] )/( delta + cte_psy*(1 + 0.34*vent[i]) ); # Equation 6 Chap 4
""" Considering product 176 = RN these equations are not needed """
Rn = Ray[i]
# Soil heat flux at daily scale
G = 0; # Equation 42 Chap 3
ET0_1[i] = ( 0.408*delta*( Rn-G )+ cte_psy*( 900/(Tmean[i] + 273) )*(es - ea)*vent[i] )/( delta + cte_psy*(1 + 0.34*vent[i]) ); # Equation 6 Chap 4
""" Considering product 176 Evaporation """
ET0_2[i] = Evap[i]
if typeOutput=='RasterFile':
#On ecrit le fichier ET0
geoTransform=utils.getGeoTransform(RayFile)
shape=Ray[0].shape
utils.writeTiffFromDicoArray(ET0_0,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/ET0.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/ET0.tif")
#On écrit le fichier Precipitation
geoTransform=utils.getGeoTransform(precipitationFile)
shape=precipitation[0].shape
utils.writeTiffFromDicoArray(precipitation,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/precipitationAcc.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/precipitationAcc.tif")
else:
#On ecrit le fichier au format Txt
proj=utils.getProj(pathToShapefile)
utils.WriteTxtFileForEachPixel(oFolder,ET0_0,ET0_1,ET0_2,DateList,DoyList,Ray,RayDownShort,RayDownLong,Tmean,Tmax,Tmin,Hmean,Hmax,Hmin,vent,precipitation,pressureMean,Geo,latlon,proj)
utils.WritePointList(oFolder,latlon,proj)
""" ------------------------------------------- """
if(temporaryFile):
#On ecrit le fichier latlon
geoTransform=utils.getGeoTransform(GeoFile)
shape=Geo[0].shape
utils.writeTiffFromDicoArray(Geo,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/altitude.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/altitude.tif")
#On ecrit le fichier latlon
geoTransform=utils.getGeoTransform(RayFile)
shape=Ray[0].shape
utils.writeTiffFromDicoArray(latlon,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/latLon.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/latLon.tif")
#On ecrit le fichier vent --> a enlever
geoTransform=utils.getGeoTransform(ventFile[-1])
shape=vent[0].shape
utils.writeTiffFromDicoArray(vent,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/ventMean.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/ventMean.tif")
#On ecrit le fichier Rhmin
geoTransform=utils.getGeoTransform(pressureFile)
shape=pressureMean[0].shape
utils.writeTiffFromDicoArray(pressureMean,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/pressureMean.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/pressureMean.tif")
#On ecrit le fichier Rhmax
geoTransform=utils.getGeoTransform(pressureFile)
shape=Hmax[0].shape
utils.writeTiffFromDicoArray(Hmax,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/humidityMax.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/humidityMax.tif")
#On ecrit le fichier Rhmin
geoTransform=utils.getGeoTransform(pressureFile)
shape=Hmin[0].shape
utils.writeTiffFromDicoArray(Hmin,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/humidityMin.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/humidityMin.tif")
#On ecrit le fichier Tmax
geoTransform=utils.getGeoTransform(T2mFile)
shape=Tmax[0].shape
utils.writeTiffFromDicoArray(Tmax,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/TemperatureMax.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/TemperatureMax.tif")
#On ecrit le fichier Tmin
geoTransform=utils.getGeoTransform(T2mFile)
shape=Tmin[0].shape
utils.writeTiffFromDicoArray(Tmin,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/TemperatureMin.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/TemperatureMin.tif")
#On ecrit le fichier Rayonnement
geoTransform=utils.getGeoTransform(RayFile)
shape=Ray[0].shape
utils.writeTiffFromDicoArray(Ray,oFolder+"/tmp.tif",shape,geoTransform)
if 'pathToShapefile' in locals():
utils.reprojRaster(oFolder+"/tmp.tif", oFolder+"/RayonnementMean.tif",shape, pathToShapefile)
os.remove(oFolder+"/tmp.tif")
else :
utils.moveFile(oFolder+"/tmp.tif", oFolder+"/RayonnementMean.tif")
#on supprime les fichier intermédiare !
os.remove(pressureFile)
os.remove(T2mFile)
os.remove(DewPFile)
os.remove(RayFile)
os.remove(GeoFile)
for i in ventFile:
os.remove(i)
os.remove(precipitationFile)
os.remove(EvapFile)
os.remove(RayFileDownLong)
os.remove(RayFileDownShort)
#!/usr/bin/env python
"""
Download ERA-Interim example dataset using the ECMWF Python API
(requires API key).
See https://software.ecmwf.int/wiki/display/WEBAPI/Access+ECMWF+Public+Datasets
"""
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date":
"19790101/19790201/19790301/19790401/19790501/19790601/19790701/19790801/19790901/19791001/19791101/19791201/19800101/19800201/19800301/19800401/19800501/19800601/19800701/19800801/19800901/19801001/19801101/19801201/19810101/19810201/19810301/19810401/19810501/19810601/19810701/19810801/19810901/19811001/19811101/19811201/19820101/19820201/19820301/19820401/19820501/19820601/19820701/19820801/19820901/19821001/19821101/19821201/19830101/19830201/19830301/19830401/19830501/19830601/19830701/19830801/19830901/19831001/19831101/19831201/19840101/19840201/19840301/19840401/19840501/19840601/19840701/19840801/19840901/19841001/19841101/19841201/19850101/19850201/19850301/19850401/19850501/19850601/19850701/19850801/19850901/19851001/19851101/19851201/19860101/19860201/19860301/19860401/19860501/19860601/19860701/19860801/19860901/19861001/19861101/19861201/19870101/19870201/19870301/19870401/19870501/19870601/19870701/19870801/19870901/19871001/19871101/19871201/19880101/19880201/19880301/19880401/19880501/19880601/19880701/19880801/19880901/19881001/19881101/19881201/19890101/19890201/19890301/19890401/19890501/19890601/19890701/19890801/19890901/19891001/19891101/19891201/19900101/19900201/19900301/19900401/19900501/19900601/19900701/19900801/19900901/19901001/19901101/19901201/19910101/19910201/19910301/19910401/19910501/19910601/19910701/19910801/19910901/19911001/19911101/19911201/19920101/19920201/19920301/19920401/19920501/19920601/19920701/19920801/19920901/19921001/19921101/19921201/19930101/19930201/19930301/19930401/19930501/19930601/19930701/19930801/19930901/19931001/19931101/19931201/19940101/19940201/19940301/19940401/19940501/19940601/19940701/19940801/19940901/19941001/19941101/19941201/19950101/19950201/19950301/19950401/19950501/19950601/19950701/19950801/19950901/19951001/19951101/19951201/19960101/19960201/19960301/19960401/19960501/19960601/19960701/19960801/19960901/19961001/19961101/19961201/19970101/19970201/19970301/19970401/19970501/19970601/19970701/19970801/19970901/19971001/19971101/19971201/19980101/19980201/19980301/19980401/19980501/19980601/19980701/19980801/19980901/19981001/19981101/19981201/19990101/19990201/19990301/19990401/19990501/19990601/19990701/19990801/19990901/19991001/19991101/19991201/20000101/20000201/20000301/20000401/20000501/20000601/20000701/20000801/20000901/20001001/20001101/20001201/20010101/20010201/20010301/20010401/20010501/20010601/20010701/20010801/20010901/20011001/20011101/20011201/20020101/20020201/20020301/20020401/20020501/20020601/20020701/20020801/20020901/20021001/20021101/20021201/20030101/20030201/20030301/20030401/20030501/20030601/20030701/20030801/20030901/20031001/20031101/20031201/20040101/20040201/20040301/20040401/20040501/20040601/20040701/20040801/20040901/20041001/20041101/20041201/20050101/20050201/20050301/20050401/20050501/20050601/20050701/20050801/20050901/20051001/20051101/20051201/20060101/20060201/20060301/20060401/20060501/20060601/20060701/20060801/20060901/20061001/20061101/20061201/20070101/20070201/20070301/20070401/20070501/20070601/20070701/20070801/20070901/20071001/20071101/20071201/20080101/20080201/20080301/20080401/20080501/20080601/20080701/20080801/20080901/20081001/20081101/20081201/20090101/20090201/20090301/20090401/20090501/20090601/20090701/20090801/20090901/20091001/20091101/20091201/20100101/20100201/20100301/20100401/20100501/20100601/20100701/20100801/20100901/20101001/20101101/20101201/20110101/20110201/20110301/20110401/20110501/20110601/20110701/20110801/20110901/20111001/20111101/20111201/20120101/20120201/20120301/20120401/20120501/20120601/20120701/20120801/20120901/20121001/20121101/20121201/20130101/20130201/20130301/20130401/20130501/20130601/20130701/20130801/20130901/20131001/20131101/20131201/20140101/20140201/20140301/20140401/20140501/20140601/20140701/20140801/20140901/20141001/20141101/20141201/20150101/20150201/20150301/20150401/20150501/20150601/20150701/20150801/20150901/20151001/20151101/20151201/20160101/20160201/20160301/20160401/20160501/20160601/20160701/20160801/20160901/20161001/20161101/20161201/20170101",
"expver": "1",
"grid": "0.75/0.75",
"levtype": "sfc",
"param": "134.128/165.128/166.128/167.128",
"stream": "moda",
"type": "an",
'format': "netcdf",
'target': "ERA-Interim-MonthlyAvg-TUVP.nc"
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "2015-01-01",
"expver": "1",
"levelist": "46/to/60",
"levtype": "sfc",
"param": "134.128/167.128",
"step": "0",
"stream": "oper",
"target": "20150101_sfc.nc",
"format": "netcdf",
"time": "00/06/12/18",
"type": "an",
"area": "75/-20/30/70",
"grid": "0.36/0.36",
})
#!/usr/bin/env python26
from ecmwfapi import ECMWFDataServer
import os
def leap_year(year):
if (year % 400 == 0):
x = 1
elif (year % 4 == 0):
x = 1
else:
x = 0
return x
server = ECMWFDataServer()
mon_name = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'
]
ord_mon = [
'31', '28', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
leap_mon = [
'31', '29', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
for year in range(1989, 2014):
for mon in range(1, 13):
# if (year<2008 and mon<10):
# continue
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "2006-06-01/to/2010-09-30",
"expver": "1",
"grid": "0.75/0.75",
"levtype": "sfc",
"param": "59.128/134.128/143.128/146.128/147.128/151.128/159.128/165.128/166.128/167.128/168.128/182.128/201.128/202.128/228.128/235.128/244.128",
"step": "6",
"stream": "oper",
"time": "12:00:00",
"type": "fc",
"area": "22/-12/8/12",
"format": "netcdf",
"target": "/localscratch/wllf030/cornkle/ERA-I/daily_2006-2010_12UTCsrfc_forecast.nc"
})
# convert the args to useful info
lonmin = args.bound[1]
lonmax = args.bound[3]
latmin = args.bound[2]
latmax = args.bound[0]
outfile = args.outfile[0]
syear = args.syear[0]
eyear = args.eyear[0]
grid = args.grid[0]
# load module
from ecmwfapi import ECMWFDataServer
# run
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "%d-01-01/to/%d-12-31" % (syear, eyear),
"expver": "1",
"grid": "%f/%f" % (grid, grid),
"levtype": "sfc",
"param": "71.162/72.162",
"step": "0",
"stream": "oper",
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
"type": "an",
"format": "netcdf", # delete this if for some reason you want GRIB...
"area":
"%f/%f/%f/%f" % (latmax, lonmin, latmin, lonmax), # NORTH/WEST/SOUTH/EAST
print " python get_erai.py <country>"
print "where <country> can be;"
print " Australia"
print " USA"
sys.exit
if sys.argv[1].lower()=="australia":
erai_info["area"] = "-10/110/-45/155"
target_directory = "/mnt/OzFlux/ERAI/"
start_year = 2014
end_year = 2016
elif sys.argv[1].lower()=="usa":
erai_info["area"] = "70/229.5/30/300"
target_directory = "/mnt/AmeriFlux/ERAI/"
start_year = 1991
end_year = 2016
else:
print "Unrecognised country option entered on command line"
print "Valid country options are:"
print " australia"
print " usa"
sys.exit()
server = ECMWFDataServer()
year_list = range(start_year,end_year,1)
for year in year_list:
print " Processing year: ",str(year)
erai_info["date"] = str(year)+"-01-01/to/"+str(year+1)+"-01-01"
erai_info["target"] = target_directory+"ERAI_"+str(year)+".nc"
server.retrieve(erai_info)
'significant_height_of_combined_wind_waves_and_swell'
],
'area': [lat_st, lon_st, lat_en, lon_en],
# 'year': '2021',
# 'month': ['01', '02'],
# 'day' : [f'{i:02d}' for i in range(1, 32)],
},
Path(dir_save) / file_nc_name
)
if False: # old
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
l.info('part 1')
server.retrieve({**common, **{
'step': '0',
'time': '00:00/06:00/12:00/18:00',
'type': 'an',
'target': file_date_prefix + 'analysis.nc',
}})
l.info('part 2')
server.retrieve({**common, **{
'step': '3/9', # '3/6/9/12'
'time': '00:00/12:00',
'type': 'fc',
'target': file_date_prefix + 'forecast.nc',
}})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
import datetime
day = datetime.datetime.today().day
month = datetime.datetime.today().month
if day < 10:
day = "0" + str(day)
else:
day = str(day)
if month < 10:
month = "0" + str(month)
else:
month = str(month)
server = ECMWFDataServer()
server.retrieve({
"class": "e2",
"dataset": "era20c",
"date": "2010-" + month + "-" + day,
"expver": "1",
"levtype": "sfc",
"param": "15.128/134.128/137.128/167.128",
"stream": "oper",
"time": "12:00:00",
"type": "an",
"target": "mars.grib",
})
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 28 16:54:48 2017
@author: sebastian
"""
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
'stream' : "oper",
'levtype' : "sfc",
'param' : "165.128/166.128",
'dataset' : "interim",
'step' : "0",
'grid' : "0.5/0.5",
'time' : "12",
'date' : "2015-01-01/to/2016-12-31",
'type' : "an",
'class' : "ei",
'area' : "50/8/49/9",
'format' : "netcdf",
'target' : "/home/sebastian/analysis_raw.nc"
})
#!/usr/bin/python
from ecmwfapi import ECMWFDataServer
import sys
server = ECMWFDataServer()
idate = sys.argv[1]
if len(idate) != 10:
raise ValueError, "get_erainterim.py idate grid"
grid = sys.argv[2]
fname = sys.argv[3]
server.retrieve({
'dataset' : "interim",
'levtype' : 'pl',
'date' : idate[:8],
'time' : idate[-2:],
'levelist': "1000/975/950/925/900/875/850/825/800/775/750/700/650/600/550/500/450/400/350/300/250/225/200/175/150/125/100/70/50/30/20/10/7/5/3/2/1",
'type' : "an",
'param' : 'z/u/v/t/q/o3/clwc/ciwc',
'grid' : '{0}/{0}'.format(grid),
'area' : "90/0/-90/360",
'target' : fname
})
#!/usr/bin/env python26
from ecmwfapi import ECMWFDataServer
import os
def leap_year(year):
if (year % 400 == 0):
x = 1
elif (year % 4 == 0):
x = 1
else:
x = 0
return x
server = ECMWFDataServer()
variables = 'Soil temperature level 1'
out_name = 'ST1'
mon_name = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'
]
ord_mon = [
'31', '28', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
leap_mon = [
'31', '29', '31', '30', '31', '30', '31', '31', '30', '31', '30', '31'
]
for year in range(2014, 2015):
for mon in range(1, 6):
from ecmwfapi import ECMWFDataServer
import numpy as np
import pdb
y = np.arange(1979,2018,1)
stri = ''
for yy in y:
stri = stri+"{0}0101/{0}0201/{0}0301/{0}0401/{0}0501/{0}0601/{0}0701/{0}0801/{0}0901/{0}1001/{0}1101/{0}1201/".format(yy)
stri = stri[0:-1]
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": stri,
"expver": "1",
"grid": "0.75/0.75",
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
"levtype": "sfc",
"param": "81.162/134.128/137.128/164.128/165.128/166.128/167.128/168.128/207.128",
"stream": "mnth",
"type": "an",
"step" : 0,
"area": "34/-23/-42/55",
"format": "netcdf",
"target": "/localscratch/wllf030/cornkle/ERA-I/monthly/monthly_synop_1979-2017_srfc_full.nc"
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ea",
"dataset": "era5",
"date": "2008-01-01/to/2008-01-31",
"expver": "1",
"levtype": "sfc",
"param": "165.128",
"grid": "0.25/0.25",
"area": "72.5/-22/26.5/45.5",
"stream": "oper",
"time":
"00:00:00/01:00:00/02:00:00/03:00:00/04:00:00/05:00:00/06:00:00/07:00:00/08:00:00/09:00:00/10:00:00/11:00:00/12:00:00/13:00:00/14:00:00/15:00:00/16:00:00/17:00:00/18:00:00/19:00:00/20:00:00/21:00:00/22:00:00/23:00:00",
"type": "an",
"format": "netcdf",
"target": "ERA5_10mUwindanalysis_2008.nc",
})
from ecmwfapi import ECMWFDataServer
import xarray as xr
import numpy as np
file = "/localscratch/wllf030/cornkle/ERA-I/daily_2004-2014_pl.nc"
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "1983-03-01/to/2014-09-30",
"expver": "1",
"grid": "0.75/0.75",
"levtype": "pl",
"levelist": "600/650/700/850/925/950",
"param": "60.128/130.128/131.128/132.128/133.128/135.128/155.128/157.128",
"step": "0",
"stream": "oper",
"time": "12:00:00",
"type": "an",
"area": "22/-15/3/15",
"format": "netcdf",
"target": file
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
import sys
str=sys.argv[1]
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": str,
"expver": "1",
"levelist": "46/to/60",
"levtype": "ml",
"param": "130.128/131.128/132.128/138.128",
"step": "0",
"stream": "oper",
"target": "2006/"+str.replace('-','')+"_ml.nc",
"format": "netcdf",
"time": "00/06/12/18",
"type": "an",
"area": "75/-20/30/70",
"grid": "0.36/0.36",
})
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "1989-01-01",
"expver": "1",
"grid": "0.25/0.25",
"levtype": "sfc",
"param": "129.128",
"step": "0",
"stream": "oper",
"target": "China.nc",
"time": "12",
"area": "60/70/10/150",
"type": "an",
"format": "netcdf",
})
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
'use':
"infrequent",
'stream':
"oper",
'dataset':
"interim",
'step':
"0",
'levtype':
"pl",
'date':
"2008-01-03/to/2008-01-03",
'time':
"00/06/12/18",
'levelist':
"750/775/800/825/850/875/900/925/950/1000",
'type':
"an",
'param':
"129.128",
'area':
"90/00/-90/360",
'grid':
"0.125/0.125",
'target':
"/home/j_timmermans/Simulations/Matlab/SEBS/SEBS4SIGMA/Data/TMP/ECMWF/Z_profile.grib"
#!/usr/bin/env python
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": "2015-01-01",
"expver": "1",
"levelist": "46/to/60",
"levtype": "ml",
"param": "131.128/132.128",
"step": "0",
"stream": "oper",
"target": "20150101_U_V_ml.nc",
"time": "00/06/12/18",
"type": "an",
"area": "60/70/10/150",
"format": "netcdf",
"grid": "0.36/0.36"
})
from multiprocessing import Pool
from math import ceil, floor
from ecmwfapi import ECMWFDataServer
import pandas as pd
from datetime import date
import helpers
import os.path
def parsedate(ts):
return date.fromtimestamp(int(ts))
server = ECMWFDataServer()
data = pd.read_csv('data/avalanches.csv',
parse_dates=['date_posix_ts'],
date_parser=parsedate)
params = []
for i in range(0, data.shape[0]):
[lat, lon] = helpers.CH1903toWGS1984(data.loc[i]['starting_zone_X'],
data.loc[i]['starting_zone_Y'])
area = str(ceil(lat * 1000) / 1000) + '/' + str(
floor(lon * 1000) / 1000) + '/' + str(
floor(lat * 1000) / 1000) + '/' + str(ceil(lon * 1000) / 1000)
target = './data/grib/avalanche' + str(i) + '.grb'
file_exists = os.path.isfile('./data/grib/avalanche' + str(i) + '.grb')
for j in range(0, 11):
file_exists |= os.path.isfile('./data/grib_batch_' + str(j) +
'/avalanche' + str(i) + '.grb')
if (not file_exists):
params.append({'area': area, 'target': target})
def ECMWF_retrieve(Hind_start, Hind_end, area_range, area_name):
"""
Function: ECMWF_retrieve(Hind_start, Hind_end, save_grib, save_dat)
List of Inputs:
Hind_start [str]: Download the ECMWF data: start date, e.g. '1999-01-01'
Hind_end [str]: Download the ECMWF data: end date, e.g. '1999-01-31'
area_range [str]: Area of MetOcean to be downloaded, "N/W/S/E, e.g. "90/-80/0/10" means North Atlantic
area_name [str]: Give a name for your downloading area
save_grid [str]: Folder to save the grid file
save_dat [str]: Folder to save the converted ASCII file
Here we need to define the following parameters:
1, The starting day of the Hindcast data downloading
2, The ending day of the Hindcast data downloading
3, The save folder of the Grib2 file from ECMWF
4, The output folder for ASCII file folder
All the input will be read from a job file.
"""
# Construct the input and output parameters
dateinterval = Hind_start + '/to/' + Hind_end
AREA = area_range
wave_grib_name = 'wave_' + area_name + Hind_start + '_' + Hind_end + '.grib'
wind_grib_name = 'wind_' + area_name + Hind_start + '_' + Hind_end + '.grib'
# Define the sever and download the data
server = ECMWFDataServer()
# Download the wave data, which will include significant wave height Hs, and wave period Tp
## 01, Define the save folder and file for downloading
wave_folder = '/Users/wengang/Documents/Python/Spatio_temporal_model/Downloaded_data/GribWaveData/'
wind_folder = '/Users/wengang/Documents/Python/Spatio_temporal_model/Downloaded_data/GribWindData/'
## 02, Define the file name and its associated folder
filename_wave = wave_folder + wave_grib_name
filename_wind = wind_folder + wind_grib_name
## 03, Download the wave data, i.e. significant wave height, mean wave period, and wave direction
params = datasets['significant_wave_height']['param'] + '/' + datasets[
'mean_wave_period']['param'] + '/' + datasets['mean_wave_dir']['param']
stream = datasets['significant_wave_height']['stream']
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": dateinterval, #"2014-07-01/to/2014-07-31",
"expver": "1",
"grid": "0.75/0.75",
"repres": "ll",
"levtype": "sfc",
"param": params,
"stream": stream,
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
"type": "an",
"area": AREA,
"target": filename_wave,
})
print "wind data is downloaded:" + wave_grib_name
## 04,Download the secondary data, such as the sea temperature, wind_U, wind_V
params = datasets['u_wind']['param'] + '/' + datasets['v_wind']['param']
#params = datasets['sea_surface_temp']['param'] + '/' + datasets['u_wind']['param'] + '/' + datasets['v_wind']['param']
stream = datasets['sea_surface_temp']['stream']
server.retrieve({
"class": "ei",
"dataset": "interim",
"date": dateinterval, #"2014-07-01/to/2014-07-31",
"expver": "1",
"grid": "0.75/0.75",
"repres": "ll",
"levtype": "sfc",
"param": params,
"stream": stream,
"time": "00:00:00/06:00:00/12:00:00/18:00:00",
"type": "an",
"area": AREA,
"target": filename_wind,
})
print "wind data is downloaded:" + wind_grib_name
from ecmwfapi import ECMWFDataServer
import numpy as np
y = np.arange(1979, 2018, 1)
stri = ''
for yy in y:
stri = stri + "{0}0101/{0}0201/{0}0301/{0}0401/{0}0501/{0}0601/{0}0701/{0}0801/{0}0901/{0}1001/{0}1101/{0}1201/".format(
yy)
stri = stri[0:-1]
server = ECMWFDataServer()
server.retrieve({
"class":
"ei",
"dataset":
"interim",
"date":
stri,
"expver":
"1",
"grid":
"0.75/0.75",
"levtype":
"pl",
"param":
"130.128/131.128/132.128/133.128/135.128/157.128", #"60.128/129.128/130.128/131.128/132.128/133.128/135.128/155.128/157.128",
"levelist":
"200/250/300/350/400/450/500/550/600/650/700/750/800/850/900/925/950",
def get(year, month, outdir='.'):
"""
Get the ECMWF ERA-20C FVCOM forcing data for a given year.
Gets the following variables at the surface and model levels.
Model:
ML_PARAMS=z/q/u/v/t/d/vo
LEVELS=1000/925/850/700/500/300/250/200/150/100/70/50/30/20/10
Surface:
SFC_PARAMS=sp/msl/skt/2t/10u/10v/2d/z/lsm/sst/ci/sd/stl1/stl2/stl3/stl4/swvl1/swvl2/swvl3/swvl4
Parameters
----------
year : int
Year for which to download data.
outdir : str, optional
Output directory for the files. Defaults to the current directory.
Returns
-------
files : tuple
File paths for the analysis and forecast data (in that order).
"""
server = ECMWFDataServer()
files = []
# Buffer by four days from the current month.
date = datetime.datetime(year, month, 01, 00, 00, 00)
dom = calendar.monthrange(year, month)[-1]
start_date = date - datetime.timedelta(days=4)
end_date = date + datetime.timedelta(dom + 4)
s_start = start_date.strftime('%Y-%m-%d')
s_end = end_date.strftime('%Y-%m-%d')
prefixes = ('{:04d}-{:02d}_ML.grb2', '{:04d}-{:02d}_SFC.grib')
files = (os.path.join(outdir, prefixes[0].format(year, month)),
os.path.join(outdir, prefixes[1].format(year, month)))
if not os.path.exists(files[0]):
try:
# Pressure levels data
server.retrieve({
"class": "e2",
"dataset": "era20c",
"date": "{}/to/{}".format(s_start, s_end),
"expver": "1",
"levelist": "1/2/3/5/7/10/20/30/50/70/100/125/150/175/200/225/250/300/350/400/450/500/550/600/650/700/750/775/800/825/850/875/900/925/950/975/1000",
"levtype": "pl",
"param": "129.128/130.128/131.128/132.128/133.128/138.128/155.128",
"stream": "oper",
"target": files[0],
"time": "00/03/06/09/12/15/18/21",
"type": "an",
})
except Exception:
os.remove(files[1])
if not os.path.exists(files[1]):
try:
# Surface data
server.retrieve({
"class": "e2",
"dataset": "era20c",
"date": "{}/to/{}".format(s_start, s_end),
"expver": "1",
"levtype": "sfc",
"param": "31.128/34.128/39.128/40.128/41.128/42.128/134.128/139.128/"
"141.128/151.128/165.128/166.128/167.128/168.128/170.128/"
"183.128/235.128/236.128",
"stream": "oper",
"target": files[1],
"time": "00/06/12/18",
"type": "an",
})
except Exception:
os.remove(files[1])
return files
from ecmwfapi import ECMWFDataServer
server = ECMWFDataServer()
server.retrieve({
"class": "ei",
"dataset": "interim",
"expver": "1",
"stream": "oper",
"type": "fc",
"levtype": "sfc",
"param": "167.128",
"date": "2017-08-01/to/2017-08-03",
"time": "00:00:00",
"step": "3",
"grid": "0.75/0.75",
"area": "75/-20/10/60",
"format": "netcdf",
"target": "test.nc"
})
