def test_domain_errors(self): pathname = test_pathname("bufr/interpreted-range.bufr") with dballe.File(pathname) as f: binmsg = next(f) with self.assertRaises(ValueError): dballe.Importer("BUFR", domain_errors="throw") importer = dballe.Importer("BUFR", domain_errors="raise") with self.assertRaises(OverflowError): importer.from_binary(binmsg) importer = dballe.Importer("BUFR", domain_errors="unset") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertFalse(val.isset) importer = dballe.Importer("BUFR", domain_errors="clamp") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertEqual(val.enqd(), 327.66) importer = dballe.Importer("BUFR", domain_errors="tag") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertEqual(val.enqd(), 327.66) a = val.enqa("B33192") self.assertTrue(a) self.assertEqual(a.enqi(), 0)
def test_create_from_msgs(self): importer = dballe.Importer("BUFR") explorer = self._explorer() with explorer.rebuild() as update: with importer.from_file(test_pathname("bufr/gts-acars-uk1.bufr")) as imp: update.add_messages(imp) explorer.set_filter({"level": dballe.Level(102, 6260000)}) self.assertCountEqual(explorer.all_stations, [ self._station("amdar", None, 48.90500, 10.63667, "EU3375"), ]) self.assertCountEqual(explorer.stations, [ self._station("amdar", None, 48.90500, 10.63667, "EU3375"), ]) self.assertEqual(explorer.all_reports, ["amdar"]) self.assertEqual(explorer.reports, ["amdar"]) self.assertEqual(explorer.all_levels, [dballe.Level(102, 6260000), None]) self.assertEqual(explorer.levels, [dballe.Level(102, 6260000)]) self.assertEqual(explorer.all_tranges, [dballe.Trange(254, 0, 0), None]) self.assertEqual(explorer.tranges, [dballe.Trange(254, 0, 0)]) self.assertCountEqual(explorer.all_varcodes, [ "B04001", "B04002", "B04003", "B04004", "B04005", "B01011", "B05001", "B06001", "B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002", ]) self.assertCountEqual(explorer.varcodes, [ "B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002", ]) self.assertEqual(explorer.all_stats, dballe.ExplorerStats(( datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 18))) self.assertEqual(explorer.stats, dballe.ExplorerStats(( datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 10)))
def testQuerySummary(self): res = {} with self.deprecated_on_db(): for result in self.db.query_summary({"query": "details"}): self.assertEqual(result["lat"], Decimal("12.34560")) self.assertEqual(result.enqi("lat"), 1234560) self.assertEqual(result.enqd("lat"), 12.34560) self.assertEqual(result.enqs("lat"), "1234560") self.assertEqual(result.enqf("lat"), "12.34560") res[(result["ana_id"], result["rep_memo"], result["level"], result["trange"], result["var"])] = (result["datetimemin"], result["datetimemax"], result["count"]) self.assertEqual( res[(1, "synop", dballe.Level(10, 11, 15, 22), dballe.Trange(20, 111, 222), 'B01011')], (datetime.datetime(1945, 4, 25, 8, 0), datetime.datetime(1945, 4, 25, 8, 0), 1)) self.assertEqual( res[(1, "synop", dballe.Level(10, 11, 15, 22), dballe.Trange(20, 111, 222), 'B01012')], (datetime.datetime(1945, 4, 25, 8, 0), datetime.datetime(1945, 4, 25, 8, 0), 1))
def assert_gts_acars_uk1_contents(self, msg): self.assertEqual(msg.get_named("year"), dballe.var("B04001", 2009)) self.assertEqual(msg.get_named("month"), dballe.var("B04002", 2)) self.assertEqual(msg.get_named("day"), dballe.var("B04003", 24)) self.assertEqual(msg.get_named("hour"), dballe.var("B04004", 11)) self.assertEqual(msg.get_named("minute"), dballe.var("B04005", 31)) self.assertEqual(msg.get_named("ident"), dballe.var("B01011", "EU3375")) self.assertEqual(msg.get_named("latitude"), dballe.var("B05001", 48.90500)) self.assertEqual(msg.get_named("longitude"), dballe.var("B06001", 10.63667)) lv = dballe.Level(102, 6260000, None, None) tr = dballe.Trange(254, 0, 0) self.assertEqual(msg.get(lv, tr, "B01006"), dballe.var("B01006", "LH968")) self.assertEqual(msg.get(lv, tr, "B02061"), dballe.var("B02061", 0)) self.assertEqual(msg.get(lv, tr, "B02062"), dballe.var("B02062", 3)) self.assertEqual(msg.get(lv, tr, "B02064"), dballe.var("B02064", 0)) self.assertEqual(msg.get(lv, tr, "B07030"), dballe.var("B07030", 6260.0)) self.assertEqual(msg.get(lv, tr, "B08004"), dballe.var("B08004", 3)) self.assertEqual(msg.get(lv, tr, "B11001"), dballe.var("B11001", 33)) self.assertEqual(msg.get(lv, tr, "B11002"), dballe.var("B11002", 33.4)) self.assertEqual(msg.get(lv, tr, "B12101"), dballe.var("B12101", 240.0)) self.assertEqual(msg.get(lv, tr, "B13002"), dballe.var("B13002", 0.0))
def test_data(self): with self.deprecated_on_db(): with self.db.query_station_data() as cur: self.assertEqual(cur.remaining, 1) for row in cur: self.assertEqual( row.data_dict, { "report": "synop", "lat": Decimal("12.34560"), "lon": Decimal("76.54320"), "B02005": 0.5, }) with self.deprecated_on_db(): with self.db.query_data({"var": "B01012"}) as cur: self.assertEqual(cur.remaining, 1) for row in cur: self.assertEqual( row.data_dict, { "report": "synop", "lat": Decimal("12.34560"), "lon": Decimal("76.54320"), "level": dballe.Level(10, 11, 15, 22), "trange": dballe.Trange(20, 111, 222), "datetime": datetime.datetime(1945, 4, 25, 8, 0), "B01012": 500, })
def test_delete_by_context_id(self): # See issue #140 records_to_del = [] with self.deprecated_on_db(): with self.db.query_data() as cur: for rec in cur: records_to_del.append(rec.query) self.assertEqual(len(records_to_del), 2) query = records_to_del[0] self.assertEqual( query, { "report": "synop", "mobile": False, "lat": Decimal("12.34560"), "lon": Decimal("76.54320"), "level": dballe.Level(10, 11, 15, 22), "trange": dballe.Trange(20, 111, 222), "datetime": datetime.datetime(1945, 4, 25, 8, 0), "var": "B01011", }) with self.deprecated_on_db(): self.db.remove_data(records_to_del[0]) with self.deprecated_on_db(): self.assertEqual(self.db.query_data().remaining, 1)
def do_qc(input_file, output_file, preserve): importer = dballe.Importer("BUFR") exporter = dballe.Exporter("BUFR") with importer.from_file(input_file) as fp: for msgs in fp: for msg in msgs: count_vars = 0 new_msg = dballe.Message("generic") new_msg.set_named("year", msg.datetime.year) new_msg.set_named("month", msg.datetime.month) new_msg.set_named("day", msg.datetime.day) new_msg.set_named("hour", msg.datetime.hour) new_msg.set_named("minute", msg.datetime.minute) new_msg.set_named("second", msg.datetime.second) new_msg.set_named("rep_memo", msg.report) new_msg.set_named("longitude", int(msg.coords[0] * 10 ** 5)) new_msg.set_named("latitude", int(msg.coords[1] * 10 ** 5)) if msg.ident: new_msg.set_named("ident", msg.ident) for data in msg.query_data({"query": "attrs"}): variable = data["variable"] attrs = variable.get_attrs() is_ok = pass_qc(attrs) v = dballe.var( data["variable"].code, data["variable"].get() ) if not is_ok: if preserve: v.seta(dballe.var("B33007", 0)) else: continue new_msg.set(data["level"], data["trange"], v) count_vars += 1 for data in msg.query_station_data({"query": "attrs"}): variable = data["variable"] attrs = variable.get_attrs() v = dballe.var( data["variable"].code, data["variable"].get() ) for a in attrs: v.seta(a) new_msg.set(dballe.Level(), dballe.Trange(), v) if count_vars > 0: output_file.write(exporter.to_binary(new_msg))
def test_iterate(self): """ Try iterating the message with cursors """ msg = dballe.Message("synop") msg.set_named("year", dballe.var("B04001", 2009)) msg.set_named("month", dballe.var("B04002", 2)) msg.set_named("day", dballe.var("B04003", 24)) msg.set_named("hour", dballe.var("B04004", 11)) msg.set_named("minute", dballe.var("B04005", 31)) msg.set_named("latitude", dballe.var("B05001", 48.90500)) msg.set_named("longitude", dballe.var("B06001", 10.63667)) msg.set(None, None, dballe.var("B01019", "Test")) lv = dballe.Level(1, 0, None, None) tr = dballe.Trange(254, 0, 0) msg.set(lv, tr, dballe.var("B11001", 33)) msg.set(lv, tr, dballe.var("B12101", 240.0)) count = 0 for cur in msg.query_stations(): self.assertEqual(cur["lat"], Decimal('48.90500')) self.assertEqual(cur["lon"], Decimal('10.63667')) count += 1 self.assertEqual(count, 1) count = 0 expected = { "B04001": 2009, "B04002": 2, "B04003": 24, "B04004": 11, "B04005": 31, "B05001": 48.90500, "B06001": 10.63667, "B01019": "Test", } for cur in msg.query_station_data(): val = expected.pop(cur["var"]) self.assertEqual(cur["variable"].enq(), val) count += 1 self.assertEqual(expected, {}) self.assertEqual(count, 8) res = [] for cur in msg.query_data(): res.append((cur["var"], cur[cur["var"]].enq())) self.assertEqual(res, [("B11001", 33), ("B12101", 240.0)])
def testFixedIndex(self): with self.db.transaction() as tr: # Ana in one dimension, network in the other query = dict(ana_id=1, rep_memo="synop", year=2007, month=1, day=1) vars = read(tr.query_data(query), (AnaIndex(), TimeRangeIndex(frozen=True, start=(dballe.Trange(4, -21600, 0), (4, -43200, 0)))), checkConflicts=False) self.assertEqual(len(vars["B13011"].dims[1]), 2) vars = read(tr.query_data(query), (AnaIndex(), TimeRangeIndex()), checkConflicts=False) self.assertEqual(len(vars["B13011"].dims[1]), 3) vars = read(tr.query_data(query), (AnaIndex(), LevelIndex(frozen=True, start=(dballe.Level(1, None, None, None),))), checkConflicts=False) self.assertEqual(len(vars["B13011"].dims[1]), 1) vars = read(tr.query_data(query), (AnaIndex(), LevelIndex()), checkConflicts=False) self.assertEqual(len(vars["B13011"].dims[1]), 2)
def test_insert_cursor_data(self): with self.another_db() as db1: with db1.transaction() as tr1: with self.deprecated_on_db(): with self.db.query_station_data() as cur: for row in cur: tr1.insert_station_data(row.data, can_add_stations=True) with self.deprecated_on_db(): with self.db.query_data() as cur: for row in cur: tr1.insert_data(row.data, can_add_stations=True) with db1.transaction() as tr: with tr.query_station_data() as cur: self.assertEqual(cur.remaining, 1) for row in cur: self.assertEqual( row.data_dict, { "report": "synop", "lat": Decimal("12.34560"), "lon": Decimal("76.54320"), "B02005": 0.5, }) with tr.query_data({"var": "B01012"}) as cur: self.assertEqual(cur.remaining, 1) for row in cur: self.assertEqual( row.data_dict, { "report": "synop", "lat": Decimal("12.34560"), "lon": Decimal("76.54320"), "level": dballe.Level(10, 11, 15, 22), "trange": dballe.Trange(20, 111, 222), "datetime": datetime.datetime( 1945, 4, 25, 8, 0), "B01012": 500, })
def test_insert_new(self): with self.transaction() as tr: with self.assertRaises(KeyError) as e: tr.insert_data({ "report": "synop", "lat": 44.5, "lon": 11.4, "level": dballe.Level(1), "trange": dballe.Trange(254), "datetime": datetime.datetime(2013, 4, 25, 12, 0, 0), "B12101": 22.4, "B12103": 17.2, }) self.assertEqual(str(e.exception), "'station not found in the database'")
def make_gts_acars_uk1_message(self): msg = dballe.Message("amdar") msg.set_named("year", dballe.var("B04001", 2009)) msg.set_named("month", dballe.var("B04002", 2)) msg.set_named("day", dballe.var("B04003", 24)) msg.set_named("hour", dballe.var("B04004", 11)) msg.set_named("minute", dballe.var("B04005", 31)) msg.set_named("ident", dballe.var("B01011", "EU3375")) msg.set_named("latitude", dballe.var("B05001", 48.90500)) msg.set_named("longitude", dballe.var("B06001", 10.63667)) lv = dballe.Level(102, 6260000, None, None) tr = dballe.Trange(254, 0, 0) msg.set(lv, tr, dballe.var("B01006", "LH968")) msg.set(lv, tr, dballe.var("B02061", 0)) msg.set(lv, tr, dballe.var("B02062", 3)) msg.set(lv, tr, dballe.var("B02064", 0)) msg.set(lv, tr, dballe.var("B07030", 6260.0)) msg.set(lv, tr, dballe.var("B08004", 3)) msg.set(lv, tr, dballe.var("B11001", 33)) msg.set(lv, tr, dballe.var("B11002", 33.4)) msg.set(lv, tr, dballe.var("B12101", 240.0)) msg.set(lv, tr, dballe.var("B13002", 0.0)) return msg
def testQueryDataCursorAccess(self): def assertResultIntEqual(result, name, value): self.assertEqual(result[name], value) self.assertEqual(result.enqi(name), value) self.assertEqual(result.enqs(name), str(value)) self.assertEqual(result.enqd(name), float(value)) self.assertEqual(result.enqf(name), str(value)) def assertResultStringEqual(result, name, value): self.assertEqual(result[name], value) self.assertRaises(RuntimeError, result.enqi, name) self.assertRaises(RuntimeError, result.enqd, name) self.assertEqual(result.enqs(name), value) self.assertEqual(result.enqf(name), value) with self.deprecated_on_db(): with self.db.query_data({ "latmin": 10.0, "var": "B01011", "query": "attrs" }) as cur: self.assertEqual(cur.remaining, 1) for result in cur: self.assertEqual(cur.remaining, 0) assertResultIntEqual(result, "priority", 101) assertResultStringEqual(result, "rep_memo", "synop") assertResultStringEqual(result, "report", "synop") # self.assertEqual(result.enqs("ana_id"), ?) self.assertIsNone(result["ident"]) self.assertRaises(RuntimeError, result.enqi, "ident") self.assertRaises(RuntimeError, result.enqd, "ident") self.assertIsNone(result.enqs("ident")) self.assertIsNone(result.enqf("ident")) self.assertIs(result["mobile"], False) self.assertEqual(result.enqi("mobile"), 0) self.assertEqual(result.enqd("mobile"), 0) self.assertEqual(result.enqs("mobile"), "0") self.assertEqual(result.enqf("mobile"), "0") self.assertEqual(result["lat"], Decimal("12.34560")) self.assertEqual(result.enqi("lat"), 1234560) self.assertEqual(result.enqd("lat"), 12.34560) self.assertEqual(result.enqs("lat"), "1234560") self.assertEqual(result.enqf("lat"), "12.34560") self.assertEqual(result["lon"], Decimal("76.54320")) self.assertEqual(result.enqi("lon"), 7654320) self.assertEqual(result.enqd("lon"), 76.54320) self.assertEqual(result.enqs("lon"), "7654320") self.assertEqual(result.enqf("lon"), "76.54320") self.assertEqual( result["coords"], (Decimal("12.34560"), Decimal("76.54320"))) self.assertRaises(RuntimeError, result.enqi, "coords") self.assertRaises(RuntimeError, result.enqd, "coords") self.assertRaises(RuntimeError, result.enqs, "coords") self.assertRaises(RuntimeError, result.enqf, "coords") self.assertEqual( result["station"], dballe.Station("synop", 12.345600, 76.543200, None)) self.assertRaises(RuntimeError, result.enqi, "station") self.assertRaises(RuntimeError, result.enqd, "station") self.assertRaises(RuntimeError, result.enqs, "station") self.assertRaises(RuntimeError, result.enqf, "station") self.assertEqual(result["datetime"], datetime.datetime(1945, 4, 25, 8, 0)) self.assertRaises(RuntimeError, result.enqi, "datetime") self.assertRaises(RuntimeError, result.enqd, "datetime") self.assertRaises(RuntimeError, result.enqs, "datetime") self.assertRaises(RuntimeError, result.enqf, "datetime") assertResultIntEqual(result, "year", 1945) assertResultIntEqual(result, "month", 4) assertResultIntEqual(result, "day", 25) assertResultIntEqual(result, "hour", 8) assertResultIntEqual(result, "min", 0) assertResultIntEqual(result, "sec", 0) self.assertEqual(result["level"], dballe.Level(10, 11, 15, 22)) self.assertRaises(RuntimeError, result.enqi, "level") self.assertRaises(RuntimeError, result.enqd, "level") self.assertRaises(RuntimeError, result.enqs, "level") self.assertRaises(RuntimeError, result.enqf, "level") assertResultIntEqual(result, "leveltype1", 10) assertResultIntEqual(result, "l1", 11) assertResultIntEqual(result, "leveltype2", 15) assertResultIntEqual(result, "l2", 22) self.assertEqual(result["trange"], dballe.Trange(20, 111, 222)) self.assertRaises(RuntimeError, result.enqi, "trange") self.assertRaises(RuntimeError, result.enqd, "trange") self.assertRaises(RuntimeError, result.enqs, "trange") self.assertRaises(RuntimeError, result.enqf, "trange") assertResultIntEqual(result, "pindicator", 20) assertResultIntEqual(result, "p1", 111) assertResultIntEqual(result, "p2", 222) assertResultStringEqual(result, "var", "B01011") # Because we use query=attrs, result["variable"] includes attributes expected = dballe.var('B01011', 'Hey Hey!!') expected.seta(dballe.var('B33007', 50)) expected.seta(dballe.var('B33036', 75)) self.assertEqual(result["variable"], expected) self.assertRaises(RuntimeError, result.enqi, "variable") self.assertRaises(RuntimeError, result.enqd, "variable") self.assertRaises(RuntimeError, result.enqs, "variable") self.assertRaises(RuntimeError, result.enqf, "variable") self.assertEqual( result["attrs"], [dballe.var('B33007', 50), dballe.var('B33036', 75)]) self.assertRaises(RuntimeError, result.enqi, "attrs") self.assertRaises(RuntimeError, result.enqd, "attrs") self.assertRaises(RuntimeError, result.enqs, "attrs") self.assertRaises(RuntimeError, result.enqf, "attrs")
def export_data(outfile, datetimemin=None, lonmin=None, latmin=None, lonmax=None, latmax=None): db = dballe.DB.connect("mem:") #db.reset() for data in iter_datastore(DATASTORE_URL): try: lon = float(data["location"]["longitude"]) except Exception as e: logging.warning(str(data["location"])) #logging.exception(e) continue try: lat = float(data["location"]["latitude"]) except Exception as e: logging.warning(str(data["location"])) #logging.exception(e) continue try: with db.transaction() as tr: tr.insert_station_data( { "lon": lon, "lat": lat, "report": "luftdaten", "B01019": str(data["location"]["id"]) }, can_add_stations=True, can_replace=True) except Exception as e: logging.exception(e) rec = {} havetowrite = False for sensordatavalues in data["sensordatavalues"]: key = sensordatavalues["value_type"] var = VARIABLE_BCODES.get(key) if var is None: logger.info( "Var for variable {} not found, skipping".format(key)) else: try: rec["lon"] = lon rec["lat"] = lat rec["report"] = "luftdaten" bcode = var["bcode"] rec[bcode] = float( sensordatavalues["value"]) * var["a"] + var["b"] rec["level"] = dballe.Level(*var["level"]) rec["trange"] = dballe.Trange(*var["trange"]) rec["datetime"] = datetime.strptime( data["timestamp"], "%Y-%m-%d %H:%M:%S") havetowrite = True except Exception as e: logging.exception(e) #rec[bcode]=None if havetowrite: try: with db.transaction() as tr: tr.insert_data(rec, can_replace=True) except Exception as e: logging.exception(e) print(rec) exporter = dballe.Exporter("BUFR") with open(outfile, "wb") as outfile: with db.transaction() as tr: for row in tr.query_messages({ "datetimemin": datetimemin, "lonmin": lonmin, "latmin": latmin, "lonmax": lonmax, "latmax": latmax }): outfile.write(exporter.to_binary(row.message))
def testCreateFull(self): t = dballe.Trange(1, 2, 3) self.assertEqual(t, (1, 2, 3)) self.assertEqual(t.pind, 1) self.assertEqual(t.p1, 2) self.assertEqual(t.p2, 3)
def testCreateEmpty(self): t = dballe.Trange(None, None, None) self.assertEqual(t, (None, None, None)) self.assertIsNone(t.pind) self.assertIsNone(t.p1) self.assertIsNone(t.p2)