def test_create_from_msgs(self):
importer = dballe.Importer("BUFR")
explorer = self._explorer()
with explorer.rebuild() as update:
with importer.from_file(test_pathname("bufr/gts-acars-uk1.bufr")) as imp:
update.add_messages(imp)
explorer.set_filter({"level": dballe.Level(102, 6260000)})
self.assertCountEqual(explorer.all_stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertCountEqual(explorer.stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertEqual(explorer.all_reports, ["amdar"])
self.assertEqual(explorer.reports, ["amdar"])
self.assertEqual(explorer.all_levels, [dballe.Level(102, 6260000), None])
self.assertEqual(explorer.levels, [dballe.Level(102, 6260000)])
self.assertEqual(explorer.all_tranges, [dballe.Trange(254, 0, 0), None])
self.assertEqual(explorer.tranges, [dballe.Trange(254, 0, 0)])
self.assertCountEqual(explorer.all_varcodes, [
"B04001", "B04002", "B04003", "B04004", "B04005", "B01011", "B05001", "B06001",
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertCountEqual(explorer.varcodes, [
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertEqual(explorer.all_stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 18)))
self.assertEqual(explorer.stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 10)))
def test_domain_errors(self):
pathname = test_pathname("bufr/interpreted-range.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
with self.assertRaises(ValueError):
dballe.Importer("BUFR", domain_errors="throw")
importer = dballe.Importer("BUFR", domain_errors="raise")
with self.assertRaises(OverflowError):
importer.from_binary(binmsg)
importer = dballe.Importer("BUFR", domain_errors="unset")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertFalse(val.isset)
importer = dballe.Importer("BUFR", domain_errors="clamp")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
importer = dballe.Importer("BUFR", domain_errors="tag")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
a = val.enqa("B33192")
self.assertTrue(a)
self.assertEqual(a.enqi(), 0)
def testQuerySummary(self):
res = {}
with self.deprecated_on_db():
for result in self.db.query_summary({"query": "details"}):
self.assertEqual(result["lat"], Decimal("12.34560"))
self.assertEqual(result.enqi("lat"), 1234560)
self.assertEqual(result.enqd("lat"), 12.34560)
self.assertEqual(result.enqs("lat"), "1234560")
self.assertEqual(result.enqf("lat"), "12.34560")
res[(result["ana_id"], result["rep_memo"], result["level"],
result["trange"],
result["var"])] = (result["datetimemin"],
result["datetimemax"], result["count"])
self.assertEqual(
res[(1, "synop", dballe.Level(10, 11, 15,
22), dballe.Trange(20, 111,
222), 'B01011')],
(datetime.datetime(1945, 4, 25, 8,
0), datetime.datetime(1945, 4, 25, 8, 0), 1))
self.assertEqual(
res[(1, "synop", dballe.Level(10, 11, 15,
22), dballe.Trange(20, 111,
222), 'B01012')],
(datetime.datetime(1945, 4, 25, 8,
0), datetime.datetime(1945, 4, 25, 8, 0), 1))
def testCreateFull(self):
lev = dballe.Level(1, 2, 3, 4)
self.assertEqual(lev, (1, 2, 3, 4))
self.assertEqual(lev, dballe.Level(1, 2, 3, 4))
self.assertEqual(lev.ltype1, 1)
self.assertEqual(lev.l1, 2)
self.assertEqual(lev.ltype2, 3)
self.assertEqual(lev.l2, 4)
def testCreateEmpty(self):
# structseq constructors have two arguments: sequence, dict=NULL)
# this is undocumented, and possibly subject to change (see
# https://bugs.python.org/issue1820)
lev = dballe.Level(None, None, None, None)
self.assertTrue(lev == lev)
self.assertTrue(lev <= lev)
self.assertEqual(lev, (None, None, None, None))
self.assertEqual(lev, dballe.Level(None, None, None, None))
self.assertIsNone(lev.ltype1)
self.assertIsNone(lev.l1)
self.assertIsNone(lev.ltype2)
self.assertIsNone(lev.l2)
def assert_gts_acars_uk1_contents(self, msg):
self.assertEqual(msg.get_named("year"), dballe.var("B04001", 2009))
self.assertEqual(msg.get_named("month"), dballe.var("B04002", 2))
self.assertEqual(msg.get_named("day"), dballe.var("B04003", 24))
self.assertEqual(msg.get_named("hour"), dballe.var("B04004", 11))
self.assertEqual(msg.get_named("minute"), dballe.var("B04005", 31))
self.assertEqual(msg.get_named("ident"),
dballe.var("B01011", "EU3375"))
self.assertEqual(msg.get_named("latitude"),
dballe.var("B05001", 48.90500))
self.assertEqual(msg.get_named("longitude"),
dballe.var("B06001", 10.63667))
lv = dballe.Level(102, 6260000, None, None)
tr = dballe.Trange(254, 0, 0)
self.assertEqual(msg.get(lv, tr, "B01006"),
dballe.var("B01006", "LH968"))
self.assertEqual(msg.get(lv, tr, "B02061"), dballe.var("B02061", 0))
self.assertEqual(msg.get(lv, tr, "B02062"), dballe.var("B02062", 3))
self.assertEqual(msg.get(lv, tr, "B02064"), dballe.var("B02064", 0))
self.assertEqual(msg.get(lv, tr, "B07030"),
dballe.var("B07030", 6260.0))
self.assertEqual(msg.get(lv, tr, "B08004"), dballe.var("B08004", 3))
self.assertEqual(msg.get(lv, tr, "B11001"), dballe.var("B11001", 33))
self.assertEqual(msg.get(lv, tr, "B11002"), dballe.var("B11002", 33.4))
self.assertEqual(msg.get(lv, tr, "B12101"),
dballe.var("B12101", 240.0))
self.assertEqual(msg.get(lv, tr, "B13002"), dballe.var("B13002", 0.0))
def test_data(self):
with self.deprecated_on_db():
with self.db.query_station_data() as cur:
self.assertEqual(cur.remaining, 1)
for row in cur:
self.assertEqual(
row.data_dict, {
"report": "synop",
"lat": Decimal("12.34560"),
"lon": Decimal("76.54320"),
"B02005": 0.5,
})
with self.deprecated_on_db():
with self.db.query_data({"var": "B01012"}) as cur:
self.assertEqual(cur.remaining, 1)
for row in cur:
self.assertEqual(
row.data_dict, {
"report": "synop",
"lat": Decimal("12.34560"),
"lon": Decimal("76.54320"),
"level": dballe.Level(10, 11, 15, 22),
"trange": dballe.Trange(20, 111, 222),
"datetime": datetime.datetime(1945, 4, 25, 8, 0),
"B01012": 500,
})
def test_delete_by_context_id(self):
# See issue #140
records_to_del = []
with self.deprecated_on_db():
with self.db.query_data() as cur:
for rec in cur:
records_to_del.append(rec.query)
self.assertEqual(len(records_to_del), 2)
query = records_to_del[0]
self.assertEqual(
query, {
"report": "synop",
"mobile": False,
"lat": Decimal("12.34560"),
"lon": Decimal("76.54320"),
"level": dballe.Level(10, 11, 15, 22),
"trange": dballe.Trange(20, 111, 222),
"datetime": datetime.datetime(1945, 4, 25, 8, 0),
"var": "B01011",
})
with self.deprecated_on_db():
self.db.remove_data(records_to_del[0])
with self.deprecated_on_db():
self.assertEqual(self.db.query_data().remaining, 1)
def do_qc(input_file, output_file, preserve):
importer = dballe.Importer("BUFR")
exporter = dballe.Exporter("BUFR")
with importer.from_file(input_file) as fp:
for msgs in fp:
for msg in msgs:
count_vars = 0
new_msg = dballe.Message("generic")
new_msg.set_named("year", msg.datetime.year)
new_msg.set_named("month", msg.datetime.month)
new_msg.set_named("day", msg.datetime.day)
new_msg.set_named("hour", msg.datetime.hour)
new_msg.set_named("minute", msg.datetime.minute)
new_msg.set_named("second", msg.datetime.second)
new_msg.set_named("rep_memo", msg.report)
new_msg.set_named("longitude", int(msg.coords[0] * 10 ** 5))
new_msg.set_named("latitude", int(msg.coords[1] * 10 ** 5))
if msg.ident:
new_msg.set_named("ident", msg.ident)
for data in msg.query_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
is_ok = pass_qc(attrs)
v = dballe.var(
data["variable"].code, data["variable"].get()
)
if not is_ok:
if preserve:
v.seta(dballe.var("B33007", 0))
else:
continue
new_msg.set(data["level"], data["trange"], v)
count_vars += 1
for data in msg.query_station_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
v = dballe.var(
data["variable"].code, data["variable"].get()
)
for a in attrs:
v.seta(a)
new_msg.set(dballe.Level(), dballe.Trange(), v)
if count_vars > 0:
output_file.write(exporter.to_binary(new_msg))
def testGhostIndexes(self):
with self.db.transaction() as tr:
# If an index rejects a variable after another index
# has successfuly added an item, we used to end up with
# a 'ghost' index entry with no items in it
indexes = (TimeRangeIndex(), LevelIndex(frozen=True, start=(dballe.Level(3, 2, None, None),)))
query = {}
query['ana_id'] = 1
query['var'] = 'B13011'
vars = read(tr.query_data(query), indexes, checkConflicts=False)
self.assertCountEqual(vars.keys(), ["B13011"])
self.assertEqual(len(vars["B13011"].dims[1]), 1)
self.assertEqual(vars["B13011"].dims[0][0], (4, -21600, 0))
def test_iterate(self):
"""
Try iterating the message with cursors
"""
msg = dballe.Message("synop")
msg.set_named("year", dballe.var("B04001", 2009))
msg.set_named("month", dballe.var("B04002", 2))
msg.set_named("day", dballe.var("B04003", 24))
msg.set_named("hour", dballe.var("B04004", 11))
msg.set_named("minute", dballe.var("B04005", 31))
msg.set_named("latitude", dballe.var("B05001", 48.90500))
msg.set_named("longitude", dballe.var("B06001", 10.63667))
msg.set(None, None, dballe.var("B01019", "Test"))
lv = dballe.Level(1, 0, None, None)
tr = dballe.Trange(254, 0, 0)
msg.set(lv, tr, dballe.var("B11001", 33))
msg.set(lv, tr, dballe.var("B12101", 240.0))
count = 0
for cur in msg.query_stations():
self.assertEqual(cur["lat"], Decimal('48.90500'))
self.assertEqual(cur["lon"], Decimal('10.63667'))
count += 1
self.assertEqual(count, 1)
count = 0
expected = {
"B04001": 2009,
"B04002": 2,
"B04003": 24,
"B04004": 11,
"B04005": 31,
"B05001": 48.90500,
"B06001": 10.63667,
"B01019": "Test",
}
for cur in msg.query_station_data():
val = expected.pop(cur["var"])
self.assertEqual(cur["variable"].enq(), val)
count += 1
self.assertEqual(expected, {})
self.assertEqual(count, 8)
res = []
for cur in msg.query_data():
res.append((cur["var"], cur[cur["var"]].enq()))
self.assertEqual(res, [("B11001", 33), ("B12101", 240.0)])
def testFixedIndex(self):
with self.db.transaction() as tr:
# Ana in one dimension, network in the other
query = dict(ana_id=1, rep_memo="synop", year=2007, month=1, day=1)
vars = read(tr.query_data(query),
(AnaIndex(), TimeRangeIndex(frozen=True, start=(dballe.Trange(4, -21600, 0), (4, -43200, 0)))),
checkConflicts=False)
self.assertEqual(len(vars["B13011"].dims[1]), 2)
vars = read(tr.query_data(query), (AnaIndex(), TimeRangeIndex()), checkConflicts=False)
self.assertEqual(len(vars["B13011"].dims[1]), 3)
vars = read(tr.query_data(query),
(AnaIndex(), LevelIndex(frozen=True, start=(dballe.Level(1, None, None, None),))),
checkConflicts=False)
self.assertEqual(len(vars["B13011"].dims[1]), 1)
vars = read(tr.query_data(query), (AnaIndex(), LevelIndex()), checkConflicts=False)
self.assertEqual(len(vars["B13011"].dims[1]), 2)
def test_insert_cursor_data(self):
with self.another_db() as db1:
with db1.transaction() as tr1:
with self.deprecated_on_db():
with self.db.query_station_data() as cur:
for row in cur:
tr1.insert_station_data(row.data,
can_add_stations=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
for row in cur:
tr1.insert_data(row.data, can_add_stations=True)
with db1.transaction() as tr:
with tr.query_station_data() as cur:
self.assertEqual(cur.remaining, 1)
for row in cur:
self.assertEqual(
row.data_dict, {
"report": "synop",
"lat": Decimal("12.34560"),
"lon": Decimal("76.54320"),
"B02005": 0.5,
})
with tr.query_data({"var": "B01012"}) as cur:
self.assertEqual(cur.remaining, 1)
for row in cur:
self.assertEqual(
row.data_dict, {
"report": "synop",
"lat": Decimal("12.34560"),
"lon": Decimal("76.54320"),
"level": dballe.Level(10, 11, 15, 22),
"trange": dballe.Trange(20, 111, 222),
"datetime": datetime.datetime(
1945, 4, 25, 8, 0),
"B01012": 500,
})
def test_insert_new(self):
with self.transaction() as tr:
with self.assertRaises(KeyError) as e:
tr.insert_data({
"report":
"synop",
"lat":
44.5,
"lon":
11.4,
"level":
dballe.Level(1),
"trange":
dballe.Trange(254),
"datetime":
datetime.datetime(2013, 4, 25, 12, 0, 0),
"B12101":
22.4,
"B12103":
17.2,
})
self.assertEqual(str(e.exception),
"'station not found in the database'")
def make_gts_acars_uk1_message(self):
msg = dballe.Message("amdar")
msg.set_named("year", dballe.var("B04001", 2009))
msg.set_named("month", dballe.var("B04002", 2))
msg.set_named("day", dballe.var("B04003", 24))
msg.set_named("hour", dballe.var("B04004", 11))
msg.set_named("minute", dballe.var("B04005", 31))
msg.set_named("ident", dballe.var("B01011", "EU3375"))
msg.set_named("latitude", dballe.var("B05001", 48.90500))
msg.set_named("longitude", dballe.var("B06001", 10.63667))
lv = dballe.Level(102, 6260000, None, None)
tr = dballe.Trange(254, 0, 0)
msg.set(lv, tr, dballe.var("B01006", "LH968"))
msg.set(lv, tr, dballe.var("B02061", 0))
msg.set(lv, tr, dballe.var("B02062", 3))
msg.set(lv, tr, dballe.var("B02064", 0))
msg.set(lv, tr, dballe.var("B07030", 6260.0))
msg.set(lv, tr, dballe.var("B08004", 3))
msg.set(lv, tr, dballe.var("B11001", 33))
msg.set(lv, tr, dballe.var("B11002", 33.4))
msg.set(lv, tr, dballe.var("B12101", 240.0))
msg.set(lv, tr, dballe.var("B13002", 0.0))
return msg
def export_data(outfile,
datetimemin=None,
lonmin=None,
latmin=None,
lonmax=None,
latmax=None):
db = dballe.DB.connect("mem:")
#db.reset()
for data in iter_datastore(DATASTORE_URL):
try:
lon = float(data["location"]["longitude"])
except Exception as e:
logging.warning(str(data["location"]))
#logging.exception(e)
continue
try:
lat = float(data["location"]["latitude"])
except Exception as e:
logging.warning(str(data["location"]))
#logging.exception(e)
continue
try:
with db.transaction() as tr:
tr.insert_station_data(
{
"lon": lon,
"lat": lat,
"report": "luftdaten",
"B01019": str(data["location"]["id"])
},
can_add_stations=True,
can_replace=True)
except Exception as e:
logging.exception(e)
rec = {}
havetowrite = False
for sensordatavalues in data["sensordatavalues"]:
key = sensordatavalues["value_type"]
var = VARIABLE_BCODES.get(key)
if var is None:
logger.info(
"Var for variable {} not found, skipping".format(key))
else:
try:
rec["lon"] = lon
rec["lat"] = lat
rec["report"] = "luftdaten"
bcode = var["bcode"]
rec[bcode] = float(
sensordatavalues["value"]) * var["a"] + var["b"]
rec["level"] = dballe.Level(*var["level"])
rec["trange"] = dballe.Trange(*var["trange"])
rec["datetime"] = datetime.strptime(
data["timestamp"], "%Y-%m-%d %H:%M:%S")
havetowrite = True
except Exception as e:
logging.exception(e)
#rec[bcode]=None
if havetowrite:
try:
with db.transaction() as tr:
tr.insert_data(rec, can_replace=True)
except Exception as e:
logging.exception(e)
print(rec)
exporter = dballe.Exporter("BUFR")
with open(outfile, "wb") as outfile:
with db.transaction() as tr:
for row in tr.query_messages({
"datetimemin": datetimemin,
"lonmin": lonmin,
"latmin": latmin,
"lonmax": lonmax,
"latmax": latmax
}):
outfile.write(exporter.to_binary(row.message))
def testQueryDataCursorAccess(self):
def assertResultIntEqual(result, name, value):
self.assertEqual(result[name], value)
self.assertEqual(result.enqi(name), value)
self.assertEqual(result.enqs(name), str(value))
self.assertEqual(result.enqd(name), float(value))
self.assertEqual(result.enqf(name), str(value))
def assertResultStringEqual(result, name, value):
self.assertEqual(result[name], value)
self.assertRaises(RuntimeError, result.enqi, name)
self.assertRaises(RuntimeError, result.enqd, name)
self.assertEqual(result.enqs(name), value)
self.assertEqual(result.enqf(name), value)
with self.deprecated_on_db():
with self.db.query_data({
"latmin": 10.0,
"var": "B01011",
"query": "attrs"
}) as cur:
self.assertEqual(cur.remaining, 1)
for result in cur:
self.assertEqual(cur.remaining, 0)
assertResultIntEqual(result, "priority", 101)
assertResultStringEqual(result, "rep_memo", "synop")
assertResultStringEqual(result, "report", "synop")
# self.assertEqual(result.enqs("ana_id"), ?)
self.assertIsNone(result["ident"])
self.assertRaises(RuntimeError, result.enqi, "ident")
self.assertRaises(RuntimeError, result.enqd, "ident")
self.assertIsNone(result.enqs("ident"))
self.assertIsNone(result.enqf("ident"))
self.assertIs(result["mobile"], False)
self.assertEqual(result.enqi("mobile"), 0)
self.assertEqual(result.enqd("mobile"), 0)
self.assertEqual(result.enqs("mobile"), "0")
self.assertEqual(result.enqf("mobile"), "0")
self.assertEqual(result["lat"], Decimal("12.34560"))
self.assertEqual(result.enqi("lat"), 1234560)
self.assertEqual(result.enqd("lat"), 12.34560)
self.assertEqual(result.enqs("lat"), "1234560")
self.assertEqual(result.enqf("lat"), "12.34560")
self.assertEqual(result["lon"], Decimal("76.54320"))
self.assertEqual(result.enqi("lon"), 7654320)
self.assertEqual(result.enqd("lon"), 76.54320)
self.assertEqual(result.enqs("lon"), "7654320")
self.assertEqual(result.enqf("lon"), "76.54320")
self.assertEqual(
result["coords"],
(Decimal("12.34560"), Decimal("76.54320")))
self.assertRaises(RuntimeError, result.enqi, "coords")
self.assertRaises(RuntimeError, result.enqd, "coords")
self.assertRaises(RuntimeError, result.enqs, "coords")
self.assertRaises(RuntimeError, result.enqf, "coords")
self.assertEqual(
result["station"],
dballe.Station("synop", 12.345600, 76.543200, None))
self.assertRaises(RuntimeError, result.enqi, "station")
self.assertRaises(RuntimeError, result.enqd, "station")
self.assertRaises(RuntimeError, result.enqs, "station")
self.assertRaises(RuntimeError, result.enqf, "station")
self.assertEqual(result["datetime"],
datetime.datetime(1945, 4, 25, 8, 0))
self.assertRaises(RuntimeError, result.enqi, "datetime")
self.assertRaises(RuntimeError, result.enqd, "datetime")
self.assertRaises(RuntimeError, result.enqs, "datetime")
self.assertRaises(RuntimeError, result.enqf, "datetime")
assertResultIntEqual(result, "year", 1945)
assertResultIntEqual(result, "month", 4)
assertResultIntEqual(result, "day", 25)
assertResultIntEqual(result, "hour", 8)
assertResultIntEqual(result, "min", 0)
assertResultIntEqual(result, "sec", 0)
self.assertEqual(result["level"],
dballe.Level(10, 11, 15, 22))
self.assertRaises(RuntimeError, result.enqi, "level")
self.assertRaises(RuntimeError, result.enqd, "level")
self.assertRaises(RuntimeError, result.enqs, "level")
self.assertRaises(RuntimeError, result.enqf, "level")
assertResultIntEqual(result, "leveltype1", 10)
assertResultIntEqual(result, "l1", 11)
assertResultIntEqual(result, "leveltype2", 15)
assertResultIntEqual(result, "l2", 22)
self.assertEqual(result["trange"],
dballe.Trange(20, 111, 222))
self.assertRaises(RuntimeError, result.enqi, "trange")
self.assertRaises(RuntimeError, result.enqd, "trange")
self.assertRaises(RuntimeError, result.enqs, "trange")
self.assertRaises(RuntimeError, result.enqf, "trange")
assertResultIntEqual(result, "pindicator", 20)
assertResultIntEqual(result, "p1", 111)
assertResultIntEqual(result, "p2", 222)
assertResultStringEqual(result, "var", "B01011")
# Because we use query=attrs, result["variable"] includes attributes
expected = dballe.var('B01011', 'Hey Hey!!')
expected.seta(dballe.var('B33007', 50))
expected.seta(dballe.var('B33036', 75))
self.assertEqual(result["variable"], expected)
self.assertRaises(RuntimeError, result.enqi, "variable")
self.assertRaises(RuntimeError, result.enqd, "variable")
self.assertRaises(RuntimeError, result.enqs, "variable")
self.assertRaises(RuntimeError, result.enqf, "variable")
self.assertEqual(
result["attrs"],
[dballe.var('B33007', 50),
dballe.var('B33036', 75)])
self.assertRaises(RuntimeError, result.enqi, "attrs")
self.assertRaises(RuntimeError, result.enqd, "attrs")
self.assertRaises(RuntimeError, result.enqs, "attrs")
self.assertRaises(RuntimeError, result.enqf, "attrs")
