def test_bufr_read(monkeypatch):
"""Test reading data and data quality on Metop-A MHS BUFR file."""
monkeypatch.setenv("BUFR_TABLES", os.path.join(test_dir, "bufrtables"))
monkeypatch.setenv("BUFR_TABLES_TYPE", "bufrdc")
from trollbufr import load_file
from trollbufr.bufr import Bufr
test_file = os.path.join(test_dir, "metop_mhs.bufr")
bufr = Bufr(os.environ["BUFR_TABLES_TYPE"], os.environ["BUFR_TABLES"])
# laod test file and iterate over BUFR
for blob, size, header in load_file.next_bufr(test_file):
# test header for first BUFR
assert header == "IEMX01 EUMP 150722"
assert size == 48598
# decode BUFR message
bufr.decode(blob)
# iterate over subsets
for report in bufr.next_subset():
i = 0
# iterate over all descriptor/data sets
for k, m, (v, q) in report.next_data():
i += 1
if i >= 4:
# after first 3 descriptor/data sets just count
continue
if i <= 3:
# type-marker for first 3 descriptor is not None
assert m is not None
continue
# assert descriptor, data value, quality
assert m is not None
assert k == 8070
assert v == 3
assert q is None
# look-up and assert name and unit
kn, ku = bufr.get_tables().lookup_elem(k)
assert kn.strip() == "TOVS/ATOVS PRODUCT QUALIFIER"
assert ku.strip() == "CODE TABLE 8070"
# assert there were 88 descriptors in the subset
assert i == 88
# leave for-loops, all tests are done
break
break
def read_bufr_to_json(args):
"""Read and decode BUFR, write as JSON formatted file.
"""
bufr = Bufr(args.tables_type, args.tables_path)
json_data = []
bufr_i = -1
for fn_in in args.in_file:
for blob, _, header in load_file.next_bufr(fn_in):
bufr_i += 1
if args.bulletin is not None and bufr_i != args.bulletin:
continue
json_data_item = {
"heading": header,
"file": os.path.basename(fn_in),
"index": bufr_i,
"status": False,
"error": None,
"bufr": None,
}
try:
json_bufr = bufr.decode(blob,
load_tables=True,
as_array=args.array)
except Exception as e:
logger.error(e,
exc_info=1 and logger.isEnabledFor(logging.DEBUG))
json_data_item["error"] = str(e)
else:
json_data_item["status"] = True
json_data_item["bufr"] = json_bufr
finally:
json_data.append(json_data_item)
import json
out_fh = open(args.out_file, "w") or sys.stdout
with out_fh as fh_out:
if args.sparse:
json.dump(json_data, fh_out)
else:
json.dump(json_data, fh_out, indent=3, separators=(',', ': '))
logging.getLogger('').addHandler(handler)
from trollbufr.bufr import Bufr
from trollbufr import load_file
import numpy as np
TESTFILE = 'TestBulletin_468'
PNGFILE = 'metopa_iasi_ctp_%s.png'
AREA = 'euro'
lon = []
lat = []
pres = []
bfr = Bufr("bufrdc", ".")
for blob, size, header in load_file.next_bufr(TESTFILE):
bfr.decode(blob)
print header, bfr.get_meta()['datetime']
for subset in bfr.next_subset():
gotit = 0
for k, m, (v, q) in subset.next_data():
if gotit:
continue
if k == 5001:
lat.append((0, 0, v))
if k == 6001:
lon.append((0, 0, v))
if k == 7004:
pres.append((0, 0, v))
gotit = 1
lons = np.concatenate(lon)
lats = np.concatenate(lat)
def read_synop(file, params, min=None, max=None):
""" Reading bufr files for synoptical station data and provide dictionary
with weather data for cloud base height and visibility.
The results are subsequently filtered by cloud base height and visibility
Arguments:
file Bufr file with synop reports
params List of parameter names that will be extracted
min Threshold for minimum value of parameter
max Threshold for maximum value of parameter
Returns list of station dictionaries for given thresholds
"""
result = {}
bfr = Bufr("libdwd", os.getenv("BUFR_TABLES"))
for blob, size, header in load_file.next_bufr(file):
bfr.decode(blob)
try:
for subset in bfr.next_subset():
stationdict = {}
for (k, m, v, q) in subset.next_data():
if k == 1015: # Station name
stationdict['name'] = v.strip()
if k == 5001: # Latitude
stationdict['lat'] = v
if k == 6001: # Longitude
stationdict['lon'] = v
if k == 7030: # Altitude
stationdict['altitude'] = v
elif k == 4001: # Year
stationdict['year'] = v
elif k == 4002: # Month
stationdict['month'] = v
elif k == 4003: # Day
stationdict['day'] = v
elif k == 4004: # Hour
stationdict['hour'] = v
elif k == 4005: # Hour
stationdict['minute'] = v
elif k == 20003: # Present weather
stationdict['present weather'] = v
# Values from 40 to 49 are refering to fog and ice fog
# Patchy fog or fog edges value 11 or 12
elif k == 20004: # Past weather
stationdict['past weather'] = v
# Values from 40 to 49 are refering to fog and ice fog
# Patchy fog or fog edges value 11 or 12
elif k == 20013: # Cloud base height
if v is not None:
if ('cbh' in stationdict.keys()
and stationdict["cbh"] is not None):
if stationdict['cbh'] > v:
stationdict['cbh'] = v
else:
stationdict['cbh'] = v
else:
stationdict['cbh'] = None
elif k == 2001: # Auto/manual measurement
# 1 - 3 : Manual human observations. Manned stations
# 0, 4 - 7 : Only automatic observations
stationdict['type'] = v
elif k == 20001: # Visibility
stationdict['visibility'] = v
elif k == 12101: # Mean air temperature in K
stationdict['air temperature'] = v
elif k == 12103: # Dew point temperature in K
stationdict['dew point'] = v
elif k == 20010: # Cloud cover in %
stationdict['cloudcover'] = v
elif k == 13003: # Relative humidity in %
stationdict['relative humidity'] = v
elif k == 11001: # Wind direction in degree
stationdict['wind direction'] = v
elif k == 11002: # Wind speed in m s-1
stationdict['wind speed'] = v
elif k == 1002: # WMO station number
stationdict['wmo'] = v
# Apply thresholds
stationtime = datetime(
stationdict['year'],
stationdict['month'],
stationdict['day'],
stationdict['hour'],
stationdict['minute'],
).strftime("%Y%m%d%H%M%S")
paralist = []
if not isinstance(params, list):
params = [params]
for param in params:
if param not in stationdict:
res = None
elif min is not None and stationdict[param] < min:
res = None
elif max is not None and stationdict[param] >= max:
res = None
elif stationdict[param] is None:
res = None
else:
res = stationdict[param]
paralist.append(res)
if all([i is None for i in paralist]):
continue
# Add station data to result list
if stationtime in result.keys():
result[stationtime].append([
stationdict['name'], stationdict['altitude'],
stationdict['lat'], stationdict['lon']
] + paralist)
else:
result[stationtime] = [[
stationdict['name'], stationdict['altitude'],
stationdict['lat'], stationdict['lon']
] + paralist]
except DummyException as e:
"ERROR: Unresolved station request: {}".format(e)
return (result)
logging.getLogger('').addHandler(handler)
from trollbufr.bufr import Bufr
from trollbufr import load_file
import numpy as np
TESTFILE = 'TestBulletin_468'
PNGFILE = 'metopa_iasi_ctp_%s.png'
AREA = 'euro'
lon = []
lat = []
pres = []
bfr = Bufr("bufrdc", ".")
for blob, size, header in load_file.next_bufr(TESTFILE):
bfr.decode(blob)
print header, bfr.get_meta()['datetime']
for subset in bfr.next_subset():
gotit = 0
for k, m, (v, q) in subset.next_data():
if gotit:
continue
if k == 5001:
lat.append((0, 0, v))
if k == 6001:
lon.append((0, 0, v))
if k == 7004:
pres.append((0, 0, v))
gotit = 1
lons = np.concatenate(lon)
lats = np.concatenate(lat)