def test_bufr_read(monkeypatch): """Test reading data and data quality on Metop-A MHS BUFR file.""" monkeypatch.setenv("BUFR_TABLES", os.path.join(test_dir, "bufrtables")) monkeypatch.setenv("BUFR_TABLES_TYPE", "bufrdc") from trollbufr import load_file from trollbufr.bufr import Bufr test_file = os.path.join(test_dir, "metop_mhs.bufr") bufr = Bufr(os.environ["BUFR_TABLES_TYPE"], os.environ["BUFR_TABLES"]) # laod test file and iterate over BUFR for blob, size, header in load_file.next_bufr(test_file): # test header for first BUFR assert header == "IEMX01 EUMP 150722" assert size == 48598 # decode BUFR message bufr.decode(blob) # iterate over subsets for report in bufr.next_subset(): i = 0 # iterate over all descriptor/data sets for k, m, (v, q) in report.next_data(): i += 1 if i >= 4: # after first 3 descriptor/data sets just count continue if i <= 3: # type-marker for first 3 descriptor is not None assert m is not None continue # assert descriptor, data value, quality assert m is not None assert k == 8070 assert v == 3 assert q is None # look-up and assert name and unit kn, ku = bufr.get_tables().lookup_elem(k) assert kn.strip() == "TOVS/ATOVS PRODUCT QUALIFIER" assert ku.strip() == "CODE TABLE 8070" # assert there were 88 descriptors in the subset assert i == 88 # leave for-loops, all tests are done break break
def read_bufr_to_json(args): """Read and decode BUFR, write as JSON formatted file. """ bufr = Bufr(args.tables_type, args.tables_path) json_data = [] bufr_i = -1 for fn_in in args.in_file: for blob, _, header in load_file.next_bufr(fn_in): bufr_i += 1 if args.bulletin is not None and bufr_i != args.bulletin: continue json_data_item = { "heading": header, "file": os.path.basename(fn_in), "index": bufr_i, "status": False, "error": None, "bufr": None, } try: json_bufr = bufr.decode(blob, load_tables=True, as_array=args.array) except Exception as e: logger.error(e, exc_info=1 and logger.isEnabledFor(logging.DEBUG)) json_data_item["error"] = str(e) else: json_data_item["status"] = True json_data_item["bufr"] = json_bufr finally: json_data.append(json_data_item) import json out_fh = open(args.out_file, "w") or sys.stdout with out_fh as fh_out: if args.sparse: json.dump(json_data, fh_out) else: json.dump(json_data, fh_out, indent=3, separators=(',', ': '))
logging.getLogger('').addHandler(handler) from trollbufr.bufr import Bufr from trollbufr import load_file import numpy as np TESTFILE = 'TestBulletin_468' PNGFILE = 'metopa_iasi_ctp_%s.png' AREA = 'euro' lon = [] lat = [] pres = [] bfr = Bufr("bufrdc", ".") for blob, size, header in load_file.next_bufr(TESTFILE): bfr.decode(blob) print header, bfr.get_meta()['datetime'] for subset in bfr.next_subset(): gotit = 0 for k, m, (v, q) in subset.next_data(): if gotit: continue if k == 5001: lat.append((0, 0, v)) if k == 6001: lon.append((0, 0, v)) if k == 7004: pres.append((0, 0, v)) gotit = 1 lons = np.concatenate(lon) lats = np.concatenate(lat)
def read_synop(file, params, min=None, max=None): """ Reading bufr files for synoptical station data and provide dictionary with weather data for cloud base height and visibility. The results are subsequently filtered by cloud base height and visibility Arguments: file Bufr file with synop reports params List of parameter names that will be extracted min Threshold for minimum value of parameter max Threshold for maximum value of parameter Returns list of station dictionaries for given thresholds """ result = {} bfr = Bufr("libdwd", os.getenv("BUFR_TABLES")) for blob, size, header in load_file.next_bufr(file): bfr.decode(blob) try: for subset in bfr.next_subset(): stationdict = {} for (k, m, v, q) in subset.next_data(): if k == 1015: # Station name stationdict['name'] = v.strip() if k == 5001: # Latitude stationdict['lat'] = v if k == 6001: # Longitude stationdict['lon'] = v if k == 7030: # Altitude stationdict['altitude'] = v elif k == 4001: # Year stationdict['year'] = v elif k == 4002: # Month stationdict['month'] = v elif k == 4003: # Day stationdict['day'] = v elif k == 4004: # Hour stationdict['hour'] = v elif k == 4005: # Hour stationdict['minute'] = v elif k == 20003: # Present weather stationdict['present weather'] = v # Values from 40 to 49 are refering to fog and ice fog # Patchy fog or fog edges value 11 or 12 elif k == 20004: # Past weather stationdict['past weather'] = v # Values from 40 to 49 are refering to fog and ice fog # Patchy fog or fog edges value 11 or 12 elif k == 20013: # Cloud base height if v is not None: if ('cbh' in stationdict.keys() and stationdict["cbh"] is not None): if stationdict['cbh'] > v: stationdict['cbh'] = v else: stationdict['cbh'] = v else: stationdict['cbh'] = None elif k == 2001: # Auto/manual measurement # 1 - 3 : Manual human observations. Manned stations # 0, 4 - 7 : Only automatic observations stationdict['type'] = v elif k == 20001: # Visibility stationdict['visibility'] = v elif k == 12101: # Mean air temperature in K stationdict['air temperature'] = v elif k == 12103: # Dew point temperature in K stationdict['dew point'] = v elif k == 20010: # Cloud cover in % stationdict['cloudcover'] = v elif k == 13003: # Relative humidity in % stationdict['relative humidity'] = v elif k == 11001: # Wind direction in degree stationdict['wind direction'] = v elif k == 11002: # Wind speed in m s-1 stationdict['wind speed'] = v elif k == 1002: # WMO station number stationdict['wmo'] = v # Apply thresholds stationtime = datetime( stationdict['year'], stationdict['month'], stationdict['day'], stationdict['hour'], stationdict['minute'], ).strftime("%Y%m%d%H%M%S") paralist = [] if not isinstance(params, list): params = [params] for param in params: if param not in stationdict: res = None elif min is not None and stationdict[param] < min: res = None elif max is not None and stationdict[param] >= max: res = None elif stationdict[param] is None: res = None else: res = stationdict[param] paralist.append(res) if all([i is None for i in paralist]): continue # Add station data to result list if stationtime in result.keys(): result[stationtime].append([ stationdict['name'], stationdict['altitude'], stationdict['lat'], stationdict['lon'] ] + paralist) else: result[stationtime] = [[ stationdict['name'], stationdict['altitude'], stationdict['lat'], stationdict['lon'] ] + paralist] except DummyException as e: "ERROR: Unresolved station request: {}".format(e) return (result)
logging.getLogger('').addHandler(handler) from trollbufr.bufr import Bufr from trollbufr import load_file import numpy as np TESTFILE = 'TestBulletin_468' PNGFILE = 'metopa_iasi_ctp_%s.png' AREA = 'euro' lon = [] lat = [] pres = [] bfr = Bufr("bufrdc", ".") for blob, size, header in load_file.next_bufr(TESTFILE): bfr.decode(blob) print header, bfr.get_meta()['datetime'] for subset in bfr.next_subset(): gotit = 0 for k, m, (v, q) in subset.next_data(): if gotit: continue if k == 5001: lat.append((0, 0, v)) if k == 6001: lon.append((0, 0, v)) if k == 7004: pres.append((0, 0, v)) gotit = 1 lons = np.concatenate(lon) lats = np.concatenate(lat)