def do_qc(input_file, output_file, preserve):
importer = dballe.Importer("BUFR")
exporter = dballe.Exporter("BUFR")
with importer.from_file(input_file) as fp:
for msgs in fp:
for msg in msgs:
count_vars = 0
new_msg = dballe.Message("generic")
new_msg.set_named("year", msg.datetime.year)
new_msg.set_named("month", msg.datetime.month)
new_msg.set_named("day", msg.datetime.day)
new_msg.set_named("hour", msg.datetime.hour)
new_msg.set_named("minute", msg.datetime.minute)
new_msg.set_named("second", msg.datetime.second)
new_msg.set_named("rep_memo", msg.report)
new_msg.set_named("longitude", int(msg.coords[0] * 10 ** 5))
new_msg.set_named("latitude", int(msg.coords[1] * 10 ** 5))
if msg.ident:
new_msg.set_named("ident", msg.ident)
for data in msg.query_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
is_ok = pass_qc(attrs)
v = dballe.var(
data["variable"].code, data["variable"].get()
)
if not is_ok:
if preserve:
v.seta(dballe.var("B33007", 0))
else:
continue
new_msg.set(data["level"], data["trange"], v)
count_vars += 1
for data in msg.query_station_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
v = dballe.var(
data["variable"].code, data["variable"].get()
)
for a in attrs:
v.seta(a)
new_msg.set(dballe.Level(), dballe.Trange(), v)
if count_vars > 0:
output_file.write(exporter.to_binary(new_msg))
def test_iterate(self):
"""
Try iterating the message with cursors
"""
msg = dballe.Message("synop")
msg.set_named("year", dballe.var("B04001", 2009))
msg.set_named("month", dballe.var("B04002", 2))
msg.set_named("day", dballe.var("B04003", 24))
msg.set_named("hour", dballe.var("B04004", 11))
msg.set_named("minute", dballe.var("B04005", 31))
msg.set_named("latitude", dballe.var("B05001", 48.90500))
msg.set_named("longitude", dballe.var("B06001", 10.63667))
msg.set(None, None, dballe.var("B01019", "Test"))
lv = dballe.Level(1, 0, None, None)
tr = dballe.Trange(254, 0, 0)
msg.set(lv, tr, dballe.var("B11001", 33))
msg.set(lv, tr, dballe.var("B12101", 240.0))
count = 0
for cur in msg.query_stations():
self.assertEqual(cur["lat"], Decimal('48.90500'))
self.assertEqual(cur["lon"], Decimal('10.63667'))
count += 1
self.assertEqual(count, 1)
count = 0
expected = {
"B04001": 2009,
"B04002": 2,
"B04003": 24,
"B04004": 11,
"B04005": 31,
"B05001": 48.90500,
"B06001": 10.63667,
"B01019": "Test",
}
for cur in msg.query_station_data():
val = expected.pop(cur["var"])
self.assertEqual(cur["variable"].enq(), val)
count += 1
self.assertEqual(expected, {})
self.assertEqual(count, 8)
res = []
for cur in msg.query_data():
res.append((cur["var"], cur[cur["var"]].enq()))
self.assertEqual(res, [("B11001", 33), ("B12101", 240.0)])
def test_set_named(self):
msg = dballe.Message("synop")
msg.set_named("year", dballe.var("B04001", 2009))
var = msg.get_named("year")
self.assertEqual(var.code, "B04001")
self.assertEqual(var.enqi(), 2009)
msg.set_named("year", 2009)
var = msg.get_named("year")
self.assertEqual(var.code, "B04001")
self.assertEqual(var.enqi(), 2009)
msg.set_named("year", 2009.0)
var = msg.get_named("year")
self.assertEqual(var.code, "B04001")
self.assertEqual(var.enqi(), 2009)
msg.set_named("year", "2009")
var = msg.get_named("year")
self.assertEqual(var.code, "B04001")
self.assertEqual(var.enqi(), 2009)
def assert_gts_acars_uk1_contents(self, msg):
self.assertEqual(msg.get_named("year"), dballe.var("B04001", 2009))
self.assertEqual(msg.get_named("month"), dballe.var("B04002", 2))
self.assertEqual(msg.get_named("day"), dballe.var("B04003", 24))
self.assertEqual(msg.get_named("hour"), dballe.var("B04004", 11))
self.assertEqual(msg.get_named("minute"), dballe.var("B04005", 31))
self.assertEqual(msg.get_named("ident"),
dballe.var("B01011", "EU3375"))
self.assertEqual(msg.get_named("latitude"),
dballe.var("B05001", 48.90500))
self.assertEqual(msg.get_named("longitude"),
dballe.var("B06001", 10.63667))
lv = dballe.Level(102, 6260000, None, None)
tr = dballe.Trange(254, 0, 0)
self.assertEqual(msg.get(lv, tr, "B01006"),
dballe.var("B01006", "LH968"))
self.assertEqual(msg.get(lv, tr, "B02061"), dballe.var("B02061", 0))
self.assertEqual(msg.get(lv, tr, "B02062"), dballe.var("B02062", 3))
self.assertEqual(msg.get(lv, tr, "B02064"), dballe.var("B02064", 0))
self.assertEqual(msg.get(lv, tr, "B07030"),
dballe.var("B07030", 6260.0))
self.assertEqual(msg.get(lv, tr, "B08004"), dballe.var("B08004", 3))
self.assertEqual(msg.get(lv, tr, "B11001"), dballe.var("B11001", 33))
self.assertEqual(msg.get(lv, tr, "B11002"), dballe.var("B11002", 33.4))
self.assertEqual(msg.get(lv, tr, "B12101"),
dballe.var("B12101", 240.0))
self.assertEqual(msg.get(lv, tr, "B13002"), dballe.var("B13002", 0.0))
def make_gts_acars_uk1_message(self):
msg = dballe.Message("amdar")
msg.set_named("year", dballe.var("B04001", 2009))
msg.set_named("month", dballe.var("B04002", 2))
msg.set_named("day", dballe.var("B04003", 24))
msg.set_named("hour", dballe.var("B04004", 11))
msg.set_named("minute", dballe.var("B04005", 31))
msg.set_named("ident", dballe.var("B01011", "EU3375"))
msg.set_named("latitude", dballe.var("B05001", 48.90500))
msg.set_named("longitude", dballe.var("B06001", 10.63667))
lv = dballe.Level(102, 6260000, None, None)
tr = dballe.Trange(254, 0, 0)
msg.set(lv, tr, dballe.var("B01006", "LH968"))
msg.set(lv, tr, dballe.var("B02061", 0))
msg.set(lv, tr, dballe.var("B02062", 3))
msg.set(lv, tr, dballe.var("B02064", 0))
msg.set(lv, tr, dballe.var("B07030", 6260.0))
msg.set(lv, tr, dballe.var("B08004", 3))
msg.set(lv, tr, dballe.var("B11001", 33))
msg.set(lv, tr, dballe.var("B11002", 33.4))
msg.set(lv, tr, dballe.var("B12101", 240.0))
msg.set(lv, tr, dballe.var("B13002", 0.0))
return msg
def on_message(client, userdata, message):
try:
m = parse_message(message.topic, message.payload)
if m is None:
return
msg = dballe.Message("generic")
if m["ident"] is not None:
msg.set_named("ident", dballe.var("B01011", m["ident"]))
msg.set_named("longitude", dballe.var("B06001", m["lon"]))
msg.set_named("latitude", dballe.var("B05001", m["lat"]))
msg.set_named("rep_memo", dballe.var("B01194", m["rep_memo"]))
if all([
m["level"] != (None, None, None, None),
m["trange"] != (None, None, None),
userdata["overwrite_date"],
]):
m["datetime"] = datetime.now()
if m["datetime"] is not None:
msg.set_named("year", dballe.var("B04001", m["datetime"].year))
msg.set_named("month", dballe.var("B04002", m["datetime"].month))
msg.set_named("day", dballe.var("B04003", m["datetime"].day))
msg.set_named("hour", dballe.var("B04004", m["datetime"].hour))
msg.set_named("minute", dballe.var("B04005", m["datetime"].minute))
msg.set_named("second", dballe.var("B04006", m["datetime"].second))
var = dballe.var(m["var"], m["value"])
for b, v in m["attributes"].items():
var.seta(dballe.var(b, v))
msg.set(m["level"], m["trange"], var)
exporter = dballe.Exporter(encoding="BUFR")
userdata["outfile"].write(exporter.to_binary(msg))
userdata["outfile"].flush()
except Exception:
import traceback
traceback.print_exc()
def testQueryDataCursorAccess(self):
def assertResultIntEqual(result, name, value):
self.assertEqual(result[name], value)
self.assertEqual(result.enqi(name), value)
self.assertEqual(result.enqs(name), str(value))
self.assertEqual(result.enqd(name), float(value))
self.assertEqual(result.enqf(name), str(value))
def assertResultStringEqual(result, name, value):
self.assertEqual(result[name], value)
self.assertRaises(RuntimeError, result.enqi, name)
self.assertRaises(RuntimeError, result.enqd, name)
self.assertEqual(result.enqs(name), value)
self.assertEqual(result.enqf(name), value)
with self.deprecated_on_db():
with self.db.query_data({
"latmin": 10.0,
"var": "B01011",
"query": "attrs"
}) as cur:
self.assertEqual(cur.remaining, 1)
for result in cur:
self.assertEqual(cur.remaining, 0)
assertResultIntEqual(result, "priority", 101)
assertResultStringEqual(result, "rep_memo", "synop")
assertResultStringEqual(result, "report", "synop")
# self.assertEqual(result.enqs("ana_id"), ?)
self.assertIsNone(result["ident"])
self.assertRaises(RuntimeError, result.enqi, "ident")
self.assertRaises(RuntimeError, result.enqd, "ident")
self.assertIsNone(result.enqs("ident"))
self.assertIsNone(result.enqf("ident"))
self.assertIs(result["mobile"], False)
self.assertEqual(result.enqi("mobile"), 0)
self.assertEqual(result.enqd("mobile"), 0)
self.assertEqual(result.enqs("mobile"), "0")
self.assertEqual(result.enqf("mobile"), "0")
self.assertEqual(result["lat"], Decimal("12.34560"))
self.assertEqual(result.enqi("lat"), 1234560)
self.assertEqual(result.enqd("lat"), 12.34560)
self.assertEqual(result.enqs("lat"), "1234560")
self.assertEqual(result.enqf("lat"), "12.34560")
self.assertEqual(result["lon"], Decimal("76.54320"))
self.assertEqual(result.enqi("lon"), 7654320)
self.assertEqual(result.enqd("lon"), 76.54320)
self.assertEqual(result.enqs("lon"), "7654320")
self.assertEqual(result.enqf("lon"), "76.54320")
self.assertEqual(
result["coords"],
(Decimal("12.34560"), Decimal("76.54320")))
self.assertRaises(RuntimeError, result.enqi, "coords")
self.assertRaises(RuntimeError, result.enqd, "coords")
self.assertRaises(RuntimeError, result.enqs, "coords")
self.assertRaises(RuntimeError, result.enqf, "coords")
self.assertEqual(
result["station"],
dballe.Station("synop", 12.345600, 76.543200, None))
self.assertRaises(RuntimeError, result.enqi, "station")
self.assertRaises(RuntimeError, result.enqd, "station")
self.assertRaises(RuntimeError, result.enqs, "station")
self.assertRaises(RuntimeError, result.enqf, "station")
self.assertEqual(result["datetime"],
datetime.datetime(1945, 4, 25, 8, 0))
self.assertRaises(RuntimeError, result.enqi, "datetime")
self.assertRaises(RuntimeError, result.enqd, "datetime")
self.assertRaises(RuntimeError, result.enqs, "datetime")
self.assertRaises(RuntimeError, result.enqf, "datetime")
assertResultIntEqual(result, "year", 1945)
assertResultIntEqual(result, "month", 4)
assertResultIntEqual(result, "day", 25)
assertResultIntEqual(result, "hour", 8)
assertResultIntEqual(result, "min", 0)
assertResultIntEqual(result, "sec", 0)
self.assertEqual(result["level"],
dballe.Level(10, 11, 15, 22))
self.assertRaises(RuntimeError, result.enqi, "level")
self.assertRaises(RuntimeError, result.enqd, "level")
self.assertRaises(RuntimeError, result.enqs, "level")
self.assertRaises(RuntimeError, result.enqf, "level")
assertResultIntEqual(result, "leveltype1", 10)
assertResultIntEqual(result, "l1", 11)
assertResultIntEqual(result, "leveltype2", 15)
assertResultIntEqual(result, "l2", 22)
self.assertEqual(result["trange"],
dballe.Trange(20, 111, 222))
self.assertRaises(RuntimeError, result.enqi, "trange")
self.assertRaises(RuntimeError, result.enqd, "trange")
self.assertRaises(RuntimeError, result.enqs, "trange")
self.assertRaises(RuntimeError, result.enqf, "trange")
assertResultIntEqual(result, "pindicator", 20)
assertResultIntEqual(result, "p1", 111)
assertResultIntEqual(result, "p2", 222)
assertResultStringEqual(result, "var", "B01011")
# Because we use query=attrs, result["variable"] includes attributes
expected = dballe.var('B01011', 'Hey Hey!!')
expected.seta(dballe.var('B33007', 50))
expected.seta(dballe.var('B33036', 75))
self.assertEqual(result["variable"], expected)
self.assertRaises(RuntimeError, result.enqi, "variable")
self.assertRaises(RuntimeError, result.enqd, "variable")
self.assertRaises(RuntimeError, result.enqs, "variable")
self.assertRaises(RuntimeError, result.enqf, "variable")
self.assertEqual(
result["attrs"],
[dballe.var('B33007', 50),
dballe.var('B33036', 75)])
self.assertRaises(RuntimeError, result.enqi, "attrs")
self.assertRaises(RuntimeError, result.enqd, "attrs")
self.assertRaises(RuntimeError, result.enqs, "attrs")
self.assertRaises(RuntimeError, result.enqf, "attrs")
