def do_qc(input_file, output_file, preserve): importer = dballe.Importer("BUFR") exporter = dballe.Exporter("BUFR") with importer.from_file(input_file) as fp: for msgs in fp: for msg in msgs: count_vars = 0 new_msg = dballe.Message("generic") new_msg.set_named("year", msg.datetime.year) new_msg.set_named("month", msg.datetime.month) new_msg.set_named("day", msg.datetime.day) new_msg.set_named("hour", msg.datetime.hour) new_msg.set_named("minute", msg.datetime.minute) new_msg.set_named("second", msg.datetime.second) new_msg.set_named("rep_memo", msg.report) new_msg.set_named("longitude", int(msg.coords[0] * 10 ** 5)) new_msg.set_named("latitude", int(msg.coords[1] * 10 ** 5)) if msg.ident: new_msg.set_named("ident", msg.ident) for data in msg.query_data({"query": "attrs"}): variable = data["variable"] attrs = variable.get_attrs() is_ok = pass_qc(attrs) v = dballe.var( data["variable"].code, data["variable"].get() ) if not is_ok: if preserve: v.seta(dballe.var("B33007", 0)) else: continue new_msg.set(data["level"], data["trange"], v) count_vars += 1 for data in msg.query_station_data({"query": "attrs"}): variable = data["variable"] attrs = variable.get_attrs() v = dballe.var( data["variable"].code, data["variable"].get() ) for a in attrs: v.seta(a) new_msg.set(dballe.Level(), dballe.Trange(), v) if count_vars > 0: output_file.write(exporter.to_binary(new_msg))
def test_iterate(self): """ Try iterating the message with cursors """ msg = dballe.Message("synop") msg.set_named("year", dballe.var("B04001", 2009)) msg.set_named("month", dballe.var("B04002", 2)) msg.set_named("day", dballe.var("B04003", 24)) msg.set_named("hour", dballe.var("B04004", 11)) msg.set_named("minute", dballe.var("B04005", 31)) msg.set_named("latitude", dballe.var("B05001", 48.90500)) msg.set_named("longitude", dballe.var("B06001", 10.63667)) msg.set(None, None, dballe.var("B01019", "Test")) lv = dballe.Level(1, 0, None, None) tr = dballe.Trange(254, 0, 0) msg.set(lv, tr, dballe.var("B11001", 33)) msg.set(lv, tr, dballe.var("B12101", 240.0)) count = 0 for cur in msg.query_stations(): self.assertEqual(cur["lat"], Decimal('48.90500')) self.assertEqual(cur["lon"], Decimal('10.63667')) count += 1 self.assertEqual(count, 1) count = 0 expected = { "B04001": 2009, "B04002": 2, "B04003": 24, "B04004": 11, "B04005": 31, "B05001": 48.90500, "B06001": 10.63667, "B01019": "Test", } for cur in msg.query_station_data(): val = expected.pop(cur["var"]) self.assertEqual(cur["variable"].enq(), val) count += 1 self.assertEqual(expected, {}) self.assertEqual(count, 8) res = [] for cur in msg.query_data(): res.append((cur["var"], cur[cur["var"]].enq())) self.assertEqual(res, [("B11001", 33), ("B12101", 240.0)])
def test_set_named(self): msg = dballe.Message("synop") msg.set_named("year", dballe.var("B04001", 2009)) var = msg.get_named("year") self.assertEqual(var.code, "B04001") self.assertEqual(var.enqi(), 2009) msg.set_named("year", 2009) var = msg.get_named("year") self.assertEqual(var.code, "B04001") self.assertEqual(var.enqi(), 2009) msg.set_named("year", 2009.0) var = msg.get_named("year") self.assertEqual(var.code, "B04001") self.assertEqual(var.enqi(), 2009) msg.set_named("year", "2009") var = msg.get_named("year") self.assertEqual(var.code, "B04001") self.assertEqual(var.enqi(), 2009)
def assert_gts_acars_uk1_contents(self, msg): self.assertEqual(msg.get_named("year"), dballe.var("B04001", 2009)) self.assertEqual(msg.get_named("month"), dballe.var("B04002", 2)) self.assertEqual(msg.get_named("day"), dballe.var("B04003", 24)) self.assertEqual(msg.get_named("hour"), dballe.var("B04004", 11)) self.assertEqual(msg.get_named("minute"), dballe.var("B04005", 31)) self.assertEqual(msg.get_named("ident"), dballe.var("B01011", "EU3375")) self.assertEqual(msg.get_named("latitude"), dballe.var("B05001", 48.90500)) self.assertEqual(msg.get_named("longitude"), dballe.var("B06001", 10.63667)) lv = dballe.Level(102, 6260000, None, None) tr = dballe.Trange(254, 0, 0) self.assertEqual(msg.get(lv, tr, "B01006"), dballe.var("B01006", "LH968")) self.assertEqual(msg.get(lv, tr, "B02061"), dballe.var("B02061", 0)) self.assertEqual(msg.get(lv, tr, "B02062"), dballe.var("B02062", 3)) self.assertEqual(msg.get(lv, tr, "B02064"), dballe.var("B02064", 0)) self.assertEqual(msg.get(lv, tr, "B07030"), dballe.var("B07030", 6260.0)) self.assertEqual(msg.get(lv, tr, "B08004"), dballe.var("B08004", 3)) self.assertEqual(msg.get(lv, tr, "B11001"), dballe.var("B11001", 33)) self.assertEqual(msg.get(lv, tr, "B11002"), dballe.var("B11002", 33.4)) self.assertEqual(msg.get(lv, tr, "B12101"), dballe.var("B12101", 240.0)) self.assertEqual(msg.get(lv, tr, "B13002"), dballe.var("B13002", 0.0))
def make_gts_acars_uk1_message(self): msg = dballe.Message("amdar") msg.set_named("year", dballe.var("B04001", 2009)) msg.set_named("month", dballe.var("B04002", 2)) msg.set_named("day", dballe.var("B04003", 24)) msg.set_named("hour", dballe.var("B04004", 11)) msg.set_named("minute", dballe.var("B04005", 31)) msg.set_named("ident", dballe.var("B01011", "EU3375")) msg.set_named("latitude", dballe.var("B05001", 48.90500)) msg.set_named("longitude", dballe.var("B06001", 10.63667)) lv = dballe.Level(102, 6260000, None, None) tr = dballe.Trange(254, 0, 0) msg.set(lv, tr, dballe.var("B01006", "LH968")) msg.set(lv, tr, dballe.var("B02061", 0)) msg.set(lv, tr, dballe.var("B02062", 3)) msg.set(lv, tr, dballe.var("B02064", 0)) msg.set(lv, tr, dballe.var("B07030", 6260.0)) msg.set(lv, tr, dballe.var("B08004", 3)) msg.set(lv, tr, dballe.var("B11001", 33)) msg.set(lv, tr, dballe.var("B11002", 33.4)) msg.set(lv, tr, dballe.var("B12101", 240.0)) msg.set(lv, tr, dballe.var("B13002", 0.0)) return msg
def on_message(client, userdata, message): try: m = parse_message(message.topic, message.payload) if m is None: return msg = dballe.Message("generic") if m["ident"] is not None: msg.set_named("ident", dballe.var("B01011", m["ident"])) msg.set_named("longitude", dballe.var("B06001", m["lon"])) msg.set_named("latitude", dballe.var("B05001", m["lat"])) msg.set_named("rep_memo", dballe.var("B01194", m["rep_memo"])) if all([ m["level"] != (None, None, None, None), m["trange"] != (None, None, None), userdata["overwrite_date"], ]): m["datetime"] = datetime.now() if m["datetime"] is not None: msg.set_named("year", dballe.var("B04001", m["datetime"].year)) msg.set_named("month", dballe.var("B04002", m["datetime"].month)) msg.set_named("day", dballe.var("B04003", m["datetime"].day)) msg.set_named("hour", dballe.var("B04004", m["datetime"].hour)) msg.set_named("minute", dballe.var("B04005", m["datetime"].minute)) msg.set_named("second", dballe.var("B04006", m["datetime"].second)) var = dballe.var(m["var"], m["value"]) for b, v in m["attributes"].items(): var.seta(dballe.var(b, v)) msg.set(m["level"], m["trange"], var) exporter = dballe.Exporter(encoding="BUFR") userdata["outfile"].write(exporter.to_binary(msg)) userdata["outfile"].flush() except Exception: import traceback traceback.print_exc()
def testQueryDataCursorAccess(self): def assertResultIntEqual(result, name, value): self.assertEqual(result[name], value) self.assertEqual(result.enqi(name), value) self.assertEqual(result.enqs(name), str(value)) self.assertEqual(result.enqd(name), float(value)) self.assertEqual(result.enqf(name), str(value)) def assertResultStringEqual(result, name, value): self.assertEqual(result[name], value) self.assertRaises(RuntimeError, result.enqi, name) self.assertRaises(RuntimeError, result.enqd, name) self.assertEqual(result.enqs(name), value) self.assertEqual(result.enqf(name), value) with self.deprecated_on_db(): with self.db.query_data({ "latmin": 10.0, "var": "B01011", "query": "attrs" }) as cur: self.assertEqual(cur.remaining, 1) for result in cur: self.assertEqual(cur.remaining, 0) assertResultIntEqual(result, "priority", 101) assertResultStringEqual(result, "rep_memo", "synop") assertResultStringEqual(result, "report", "synop") # self.assertEqual(result.enqs("ana_id"), ?) self.assertIsNone(result["ident"]) self.assertRaises(RuntimeError, result.enqi, "ident") self.assertRaises(RuntimeError, result.enqd, "ident") self.assertIsNone(result.enqs("ident")) self.assertIsNone(result.enqf("ident")) self.assertIs(result["mobile"], False) self.assertEqual(result.enqi("mobile"), 0) self.assertEqual(result.enqd("mobile"), 0) self.assertEqual(result.enqs("mobile"), "0") self.assertEqual(result.enqf("mobile"), "0") self.assertEqual(result["lat"], Decimal("12.34560")) self.assertEqual(result.enqi("lat"), 1234560) self.assertEqual(result.enqd("lat"), 12.34560) self.assertEqual(result.enqs("lat"), "1234560") self.assertEqual(result.enqf("lat"), "12.34560") self.assertEqual(result["lon"], Decimal("76.54320")) self.assertEqual(result.enqi("lon"), 7654320) self.assertEqual(result.enqd("lon"), 76.54320) self.assertEqual(result.enqs("lon"), "7654320") self.assertEqual(result.enqf("lon"), "76.54320") self.assertEqual( result["coords"], (Decimal("12.34560"), Decimal("76.54320"))) self.assertRaises(RuntimeError, result.enqi, "coords") self.assertRaises(RuntimeError, result.enqd, "coords") self.assertRaises(RuntimeError, result.enqs, "coords") self.assertRaises(RuntimeError, result.enqf, "coords") self.assertEqual( result["station"], dballe.Station("synop", 12.345600, 76.543200, None)) self.assertRaises(RuntimeError, result.enqi, "station") self.assertRaises(RuntimeError, result.enqd, "station") self.assertRaises(RuntimeError, result.enqs, "station") self.assertRaises(RuntimeError, result.enqf, "station") self.assertEqual(result["datetime"], datetime.datetime(1945, 4, 25, 8, 0)) self.assertRaises(RuntimeError, result.enqi, "datetime") self.assertRaises(RuntimeError, result.enqd, "datetime") self.assertRaises(RuntimeError, result.enqs, "datetime") self.assertRaises(RuntimeError, result.enqf, "datetime") assertResultIntEqual(result, "year", 1945) assertResultIntEqual(result, "month", 4) assertResultIntEqual(result, "day", 25) assertResultIntEqual(result, "hour", 8) assertResultIntEqual(result, "min", 0) assertResultIntEqual(result, "sec", 0) self.assertEqual(result["level"], dballe.Level(10, 11, 15, 22)) self.assertRaises(RuntimeError, result.enqi, "level") self.assertRaises(RuntimeError, result.enqd, "level") self.assertRaises(RuntimeError, result.enqs, "level") self.assertRaises(RuntimeError, result.enqf, "level") assertResultIntEqual(result, "leveltype1", 10) assertResultIntEqual(result, "l1", 11) assertResultIntEqual(result, "leveltype2", 15) assertResultIntEqual(result, "l2", 22) self.assertEqual(result["trange"], dballe.Trange(20, 111, 222)) self.assertRaises(RuntimeError, result.enqi, "trange") self.assertRaises(RuntimeError, result.enqd, "trange") self.assertRaises(RuntimeError, result.enqs, "trange") self.assertRaises(RuntimeError, result.enqf, "trange") assertResultIntEqual(result, "pindicator", 20) assertResultIntEqual(result, "p1", 111) assertResultIntEqual(result, "p2", 222) assertResultStringEqual(result, "var", "B01011") # Because we use query=attrs, result["variable"] includes attributes expected = dballe.var('B01011', 'Hey Hey!!') expected.seta(dballe.var('B33007', 50)) expected.seta(dballe.var('B33036', 75)) self.assertEqual(result["variable"], expected) self.assertRaises(RuntimeError, result.enqi, "variable") self.assertRaises(RuntimeError, result.enqd, "variable") self.assertRaises(RuntimeError, result.enqs, "variable") self.assertRaises(RuntimeError, result.enqf, "variable") self.assertEqual( result["attrs"], [dballe.var('B33007', 50), dballe.var('B33036', 75)]) self.assertRaises(RuntimeError, result.enqi, "attrs") self.assertRaises(RuntimeError, result.enqd, "attrs") self.assertRaises(RuntimeError, result.enqs, "attrs") self.assertRaises(RuntimeError, result.enqf, "attrs")