def retrieve_from_mars(notification):
print(f"Notification for step {notification['request']['step']} received")
# now do a MARS request with this notification...
mars_server = ECMWFService("mars")
request = notification['request']
# extend the notification with the attributes needed
request.update({
"type": "fc",
"levtype": "sfc",
"param": 167.128,
"area": "75/-20/10/60"
})
mars_server.execute(request, "my_data.grib")
def download_forecast(date, hours=11):
"""
Downloads forecast data starting from the given time stamp.
Files are stored to disk in 'ecmwf_YYYY-MM-DDTHH_XXh.nc' format.
YYYY-MM-DDTHH - forecast start date
XX - number of hours
:arg date: a datetime object, e.g. datetime(2019, 5, 1, 12)
:kwarg int hours: number of hourly steps to download, e.g. 12
"""
with open('mars-credentials.json', 'r') as f:
d = json.load(f)
key = d['key']
email = d['email']
server = ECMWFService('mars',
url='https://api.ecmwf.int/v1',
key='80b00604dbf257efd28aa352edb70e32',
email='*****@*****.**')
step_str = '/'.join(map(str, numpy.arange(int(hours + 1))))
output_file = 'ecmwf_{:%Y-%m-%dT%H}_{:02d}h.nc'.format(date, hours)
request = {
'class': 'od',
'date': date.strftime('%Y-%m-%d'),
'expver': '73',
'levtype': 'sfc',
'param':
'134.128/144.128/165.128/166.128/167.128/168.128/169.128/175.128/228.128',
'step': step_str,
'stream': 'oper',
'area': '66/-5/48/31',
'grid': '0.20689655172413793/0.1125',
'format': 'netcdf',
'time': date.strftime('%H:%M:%S'),
'type': 'fc',
}
# download to tmp file
tmpfile = 'tmp_' + output_file
server.execute(request, tmpfile)
# copy to convert all variables to float datatype
concat_netcdf_files([tmpfile],
output_file,
rm_source_files=True,
verbose=False)
# deaccumulate accumulated fields
deaccumulate_fields(output_file, ['ssrd', 'strd', 'tp', 'sf'])
# compute specific humidity
compute_specific_humidity(output_file)
return output_file
#!/usr/bin/env python
from ecmwfapi import ECMWFService
import calendar
server = ECMWFService("mars")
for year in range(1982, 2018):
#Select year for data extraction
print ('YEAR ',year)
initmonth=8
initbdate="%s%02d01"%(year,initmonth)
print ("######### ERA-interim #########")
print ('get data from ', initbdate)
print ("################################")
server.execute({
'class' : "od",
'step' : "2904/to/5160/by/24",
'number' : "0/to/50",
'levtype' : "sfc",
'expver' : "1",
'method' : "1",
'origin' : "ecmf",
'date' : "%s"%(initbdate),
'time' : "00:00:00",
'type' : "fc",
'param' : "34.128",
'stream' : "mmsf",
'system' : "4",
'grid' : "128"},
"/storage/silver/acrcc/vg140344/HGT_S4Hindcasts_0108_SST_24Hourly_%s.grib"%(initbdate))
from ecmwfapi import ECMWFService
import pandas as pd
server = ECMWFService("mars")
date_end = pd.date_range(start='2012-01-01', end='2020-01-01', freq='M')
date_start = pd.date_range(start='2012-01-01', end='2019-12-02', freq='MS')
for x, y in zip(date_start, date_end):
date = x.strftime('%Y-%m-%d') + '/to/' + y.strftime('%Y-%m-%d')
datestemp = date[14:21]
target = 'Ctrl6_%s.nc' % (datestemp)
server.execute(
{
"class": "od",
"stream": "enfo",
"expver": "1",
"type": "cf",
"levtype": "sfc",
"time": "00",
"step": "6",
"date": date,
"param": "228.128",
"grid": "0.25/0.25",
"format": "netcdf",
}, target)
step, output = pair
request = {
"class": "od",
"date": '2018-02-01/to/2018-04-01',
"expver": 1,
"grid": "0.125/0.125",
"resol": "av",
"area": "90/-180/40/180",
"levtype": "sfc",
"param": "10u/10v/2t/2d/tcc/tp/msl/ssrd/strd/sf",
"step": step,
"stream": "oper",
"time": "00",
"type": "fc",
"format": "netcdf",
}
print(json.dumps(request, indent=4))
print('Output file: %s' % output)
server.execute(request, output)
server = ECMWFService('mars')
pairs = [
("0/to/21/by/3", 'legacy_201803_3h.1.nc'),
("24/to/45/by/3", 'legacy_201803_3h.2.nc'),
("48/to/69/by/3", 'legacy_201803_3h.3.nc'),
("72/to/93/by/3", 'legacy_201803_3h.4.nc'),
]
p = Pool(4)
p.map(download, pairs)
To find observations online:
Mars Catalogue / Operational Archive / Atmospheric Model / 1 / Observations-Observations / Year / Month / Conventional Data /
select day, time and observation type (scroll to Synop)
'''
import os
import datetime as dt
import calendar
import numpy as np
from ecmwfapi import ECMWFService
server = ECMWFService("mars") # want the MARS archive
# JASMIN group work space
GWS = "/group_workspaces/jasmin2/c3s311a_lot2/data/incoming/"
outdir = "mars/"
version = "v20170628/"
"""
OBSTYPE:
https://software.ecmwf.int/wiki/display/UDOC/Identification+keywords#Identificationkeywords-obstype
Land surface and sea surface data types are:
LSD - 1/2/3/4/140/
SSD - 9/11/12/13/14/19/20/21/22/23/
Liz Kent said not to worry about Reduced COADS data - #20
# parameters
# phis/lnps/tmp/spfh/ugrd/vgrd/o3mr
# "129/152/130/133/131/132/203"
# number of vertical levels.
nlevs = 137
levelist = []
for nlev in range(1, nlevs + 1):
levelist.append('%s/' % nlev)
levelist137 = ''.join(levelist)[:-1]
# output grid
grid = 'F400' # 800 latitude gaussian grid. F1280,F640 also supported.
# get data one date and field at a time.
server = ECMWFService('mars')
for date in daterange(date1, date2, hrinc):
yyyy, mm, dd, hh = splitdate(date)
print "requesting 2d fields for %s ...." % date
server.execute(
{
'stream': "oper",
'class': "od",
'expver': "69",
'levtype': "ml",
'type': "an",
'param': "129/152",
'date': "%04i-%02i-%02i" % (yyyy, mm, dd),
'time': "%02i" % hh,
'type': "an",
'levelist': "1",
#!/usr/bin/env python
from ecmwfapi import ECMWFService
import calendar
server = ECMWFService("mars")
for year in [1981, 1982, 2017]:
#Select year for data extraction
print ('YEAR ',year)
initmonth=11
initbdate="%s%02d01"%(year,initmonth)
print ("######### ERA-interim #########")
print ('get data from ', initbdate)
print ("################################")
server.execute({
'class' : "od",
'levelist': "200",
'step' : "0/to/2928/by/24",
'number' : "0/to/50",
'levtype' : "pl",
'expver' : "1",
'method' : "1",
'origin' : "ecmf",
'date' : "%s"%(initbdate),
'time' : "00:00:00",
'type' : "fc",
'param' : "129.128",
'stream' : "mmsf",
'system' : "4",
#!/usr/bin/env python3
from ecmwfapi import ECMWFService
server = ECMWFService("mars")
for month in [11, 12, 1, 2, 3]:
for year in range(1981, 2020):
print(f"Retrieving SEAS5, year = {year}, init_date = {month}")
start_date = f"{year}-{month:02}-01"
server.execute(
{
"class": "od",
"levelist": "500",
"step": "0/to/3648/by/24",
"number": "0",
"levtype": "pl",
"expver": "1",
"method": "1",
"origin": "ecmf",
"date": start_date,
"time": "00:00:00",
"type": "fc",
"param": "131",
"stream": "mmsf",
"system": "5",
"grid": "128",
"format": "netcdf",
},
def pull(self, archive, product):
from ecmwfapi import ECMWFDataServer, ECMWFService
dataserver = ECMWFDataServer(log=logging.info)
marsservice = ECMWFService("mars", log=logging.info)
if getattr(product.core, "archive_path", None) is None:
raise Error("cannot pull files that do not have archive_path set")
# Determine the (absolute) path in the archive that will contain the product and create it if required.
abs_archive_path = os.path.realpath(
os.path.join(archive._root, product.core.archive_path))
abs_product_path = os.path.join(abs_archive_path,
product.core.physical_name)
# Create destination location for product
try:
util.make_path(abs_archive_path)
except EnvironmentError as _error:
raise Error("cannot create parent destination path '%s' [%s]" %
(abs_archive_path, _error))
requests = []
for order in product.core.remote_url.split('?')[1].split(
'&concatenate&'):
request = {}
for param in order.split('&'):
key, value = param.split('=')
request[key] = value
requests.append(request)
plugin = archive.product_type_plugin(product.core.product_type)
# Create a temporary directory and download the product there, then move the product to its
# destination within the archive.
try:
with util.TemporaryDirectory(prefix=".pull-",
suffix="-%s" % product.core.uuid.hex,
dir=abs_archive_path) as tmp_path:
# Create enclosing directory if required.
if plugin.use_enclosing_directory:
tmp_path = os.path.join(tmp_path,
product.core.physical_name)
util.make_path(tmp_path)
# Download product.
tmp_file_combined = os.path.join(tmp_path,
product.core.physical_name)
tmp_file_download = os.path.join(tmp_path, "request.grib")
combined_file = open(tmp_file_combined, "w")
for request in requests:
if 'dataset' in request:
request['target'] = tmp_file_download
dataserver.retrieve(request)
else:
marsservice.execute(request, tmp_file_download)
result_file = open(tmp_file_download, "r")
combined_file.write(result_file.read())
result_file.close()
os.remove(tmp_file_download)
combined_file.close()
# Move the retrieved product into its destination within the archive.
if plugin.use_enclosing_directory:
os.rename(tmp_path, abs_product_path)
else:
os.rename(tmp_file_combined, abs_product_path)
except EnvironmentError as _error:
raise Error(
"unable to transfer product to destination path '%s' [%s]" %
(abs_product_path, _error))
#!/usr/bin/env python
from ecmwfapi import ECMWFService
import calendar
server = ECMWFService("mars")
for year in range(1981, 2018):
#Select year for data extraction
print ('YEAR ',year)
initmonth = 8
initbdate = "%s%02d01"%(year,initmonth)
print ("######### ERA-interim #########")
print ('get data from ', initbdate)
print ("################################")
server.execute({
'class' : "od",
'levelist': "1/2/5/10/20/30/50/70/100/150/200/250/300/400/500/700/850/925/1000",
'fcmonth' : "1/2/3/4/5/6/7",
'number' : "0/to/50",
'levtype' : "pl",
'expver' : "1",
'method' : "1",
'origin' : "ecmf",
'date' : "%s"%(initbdate),
'time' : "00:00:00",
'type' : "fcmean",
'param' : "129.128",
'stream' : "msmm",
'system' : "4",
'grid' : "128",},
#!/usr/bin/env python
from ecmwfapi import ECMWFService
import calendar
server = ECMWFService("mars")
for year in range(1982, 2018):
#Select year for data extraction
print('YEAR ', year)
initmonth = 8
initbdate = "%s%02d01" % (year, initmonth)
print("######### ERA-interim #########")
print('get data from ', initbdate)
print("################################")
server.execute(
{
'class': "od",
'levelist': "50",
'step': "0/to/5160/by/24",
'number': "0/to/50",
'levtype': "pl",
'expver': "1",
'method': "1",
'origin': "ecmf",
'date': "%s" % (initbdate),
'time': "00:00:00",
'type': "fc",
'param': "131/132",
'stream': "mmsf",
'system': "4",
#!/usr/bin/env python
from ecmwfapi import ECMWFService
server = ECMWFService("mars",
url="https://api.ecmwf.int/v1",
key="9182bc70a68dd50cee8ee720d3d69bc8",
email="*****@*****.**")
# To run this example, you need an API key
# available from https://api.ecmwf.int/v1/key/
server.execute({
'class': 'od',
'stream': 'oper',
'grid': '1/1',
'param': 't'
},
target='t.grib')
## PARA DADOS EM NÍVEL DE SUPERFÍCIE
#imports
import pandas as pd
from datetime import datetime, timedelta
from ecmwfapi import ECMWFService
#fazendo um range para se colocar às datas selecionadas
date_range = pd.date_range(start='20210801', end='20210801',
freq="D").strftime('%Y%m%d').to_list()
for dates in date_range:
server = ECMWFService("mars",
url="https://api.ecmwf.int/v1",
key="60f7f41f253aef87cd57738132d2e634",
email="*****@*****.**")
server.execute(
{
"class": "od",
"date":
dates, #pode remover esse range e usar dada como 20160101/to/20160102
"expver": "1",
"levtype": "sfc",
"number": "0/1/2/3/4/5/6/7/8/9/10/11/12/13/14",
"param": "228.128", #Buscar parâmetro da variável no catálogo
"grid": "0.2/0.2", #tamanho da grade
"step": "all", #step de horas
"area": "-25.0/-54.4/-26.8/-49.0", #lat/lon
"stream": "mmsf",
"time": "00", #rodada
"type": "fc",
"target": "data.grib2"
},
"../Dados/ECMWF_Grib/teste_ECMWF_anual.grib2")
levelist = []
for nlev in range(1, nlevs + 1):
levelist.append('%s/' % nlev)
levelist137 = ''.join(levelist)[:-1]
# output grid
#grid = '0.075/0.075' # lat/lon spacing
grid = 'F640' # 1280 latitude gaussian grid
#parameters="phis/lnps/tmp/spfh/ugrd/vgrd/clwmr/icwmr/rnwmr/snwmr/vvel"
#parameters="129/152/130/133/131/132/246/247/75/76/135"
# phis/lnps/tmp/spfh/ugrd/vgrd/o3mr
parameters = "129/152/130/133/131/132/203"
# get data one date at a time, all fields at once.
server = ECMWFService('mars')
yyyy, mm, dd, hh = splitdate(date)
print "requesting fields for %s ...." % date
server.execute(
{
'stream': "oper",
'class': "od",
'expver': "1",
'levtype': "ml",
'type': "an",
'param': "%s" % parameters,
'date': "%04i-%02i-%02i" % (yyyy, mm, dd),
'time': "%02i" % hh,
'levelist': "%s" % levelist137,
'format': "netcdf",
from ecmwfapi import ECMWFService
import pandas as pd
server = ECMWFService("mars")
date_end = pd.date_range(start='2012-01-01', end='2020-01-01', freq='M')
date_start = pd.date_range(start='2012-01-01', end='2019-12-02', freq='MS')
for x, y in zip(date_start, date_end):
date = x.strftime('%Y-%m-%d') + '/to/' + y.strftime('%Y-%m-%d')
datestemp = date[14:21]
target = 'Prtl6_%s.nc' % (datestemp)
server.execute(
{
"class": "od",
"stream": "enfo",
"expver": "1",
"type": "pf",
"levtype": "sfc",
"number":
"1/2/3/4/5/6/7/8/9/10/11/12/13/14/15/16/17/18/19/20/21/22/23/24/25/26/27/28/29/30/31/32/33/34/35/36/37/38/39/40/41/42/43/44/45/46/47/48/49/50",
"time": "00",
"step": "6",
"date": date,
"param": "228.128",
"grid": "0.25/0.25",
"format": "netcdf",
}, target)
