def test_domain_errors(self):
pathname = test_pathname("bufr/interpreted-range.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
with self.assertRaises(ValueError):
dballe.Importer("BUFR", domain_errors="throw")
importer = dballe.Importer("BUFR", domain_errors="raise")
with self.assertRaises(OverflowError):
importer.from_binary(binmsg)
importer = dballe.Importer("BUFR", domain_errors="unset")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertFalse(val.isset)
importer = dballe.Importer("BUFR", domain_errors="clamp")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
importer = dballe.Importer("BUFR", domain_errors="tag")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
a = val.enqa("B33192")
self.assertTrue(a)
self.assertEqual(a.enqi(), 0)
def test_issue216(self):
"""
Check that one can import messages with either all data, station data
only, measured data only
"""
with self._explorer("e1") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages)
self.assertEqual(explorer.all_varcodes, [
'B01011', 'B02062', 'B02064', 'B04001', 'B04002', 'B04003',
'B04004', 'B04005', 'B04006', 'B05001', 'B06001', 'B07030',
'B08004', 'B11001', 'B11002', 'B12101', 'B13002'
])
with self._explorer("e2") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=True,
data=False)
self.assertEqual(explorer.all_varcodes, [
'B01011', 'B04001', 'B04002', 'B04003', 'B04004', 'B04005',
'B04006', 'B05001', 'B06001'
])
with self._explorer("e3") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=False,
data=True)
self.assertEqual(explorer.all_varcodes, [
'B02062', 'B02064', 'B07030', 'B08004', 'B11001', 'B11002',
'B12101', 'B13002'
])
with self._explorer("e4") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=False,
data=False)
self.assertEqual(explorer.all_varcodes, [])
def test_create_from_msgs(self):
importer = dballe.Importer("BUFR")
explorer = self._explorer()
with explorer.rebuild() as update:
with importer.from_file(test_pathname("bufr/gts-acars-uk1.bufr")) as imp:
update.add_messages(imp)
explorer.set_filter({"level": dballe.Level(102, 6260000)})
self.assertCountEqual(explorer.all_stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertCountEqual(explorer.stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertEqual(explorer.all_reports, ["amdar"])
self.assertEqual(explorer.reports, ["amdar"])
self.assertEqual(explorer.all_levels, [dballe.Level(102, 6260000), None])
self.assertEqual(explorer.levels, [dballe.Level(102, 6260000)])
self.assertEqual(explorer.all_tranges, [dballe.Trange(254, 0, 0), None])
self.assertEqual(explorer.tranges, [dballe.Trange(254, 0, 0)])
self.assertCountEqual(explorer.all_varcodes, [
"B04001", "B04002", "B04003", "B04004", "B04005", "B01011", "B05001", "B06001",
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertCountEqual(explorer.varcodes, [
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertEqual(explorer.all_stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 18)))
self.assertEqual(explorer.stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 10)))
def test_refcounts(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
self.assertEqual(sys.getrefcount(importer), 2) # importer, getrefcount
with dballe.File(pathname) as f:
self.assertEqual(sys.getrefcount(importer),
2) # importer, getrefcount
self.assertEqual(sys.getrefcount(f),
3) # __enter__ result, f, getrefcount
fimp = importer.from_file(f)
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f),
4) # __enter__ result, f, fimp, getrefcount
decoded = list(fimp)
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f),
4) # __enter__ result, f, fimp, getrefcount
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f), 3) # f, fimp, getrefcount
self.assertEqual(len(decoded), 1)
def test_import_message_sequence(self):
importer = dballe.Importer("BUFR")
with dballe.File(test_pathname("bufr/vad.bufr")) as fp:
for binmsg in fp:
self.db.import_messages(importer.from_binary(binmsg))
with self.deprecated_on_db():
self.assertEqual(self.db.query_data({}).remaining, 371)
def fill_data_db(self, rec, memdb):
query = self.record_to_arkiquery(rec)
with io.BytesIO() as stdoutbytesio:
with redirect_stdout(stdoutbytesio):
sys.argv = [
"borinud", "--data", "--config", self.dataset, query
]
Query.main()
stdoutbytesio.seek(0, 0)
importer = dballe.Importer("BUFR")
with importer.from_file(stdoutbytesio) as f:
with memdb.transaction() as tr:
for msgs in f:
for msg in msgs:
try:
tr.import_messages(msg,
overwrite=True,
update_station=True,
import_attributes=True)
except:
sys.stderr.write(
"ERROR {m.report},{m.coords},{m.ident},{m.datetime},{m.type}"
.format(m=msg))
def test_default(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
importer = dballe.Importer("BUFR")
msg = importer.from_binary(binmsg)
self.assertEqual(len(msg), 1)
self.assert_gts_acars_uk1_contents(msg[0])
def test_import_varlist(self):
with self.transaction() as tr:
tr.remove_all()
importer = dballe.Importer("BUFR")
with dballe.File(test_pathname("bufr/vad.bufr")) as fp:
tr.import_messages(importer.from_file(fp),
varlist="B11001,B11002")
for cur in tr.query_data():
self.assertIn(cur["var"], ("B11001", "B11002"))
def test_fromfile_shortcut_pathname(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
with importer.from_file(pathname) as f:
decoded = list(f)
self.assertEqual(len(decoded), 1)
self.assertEqual(len(decoded[0]), 1)
msg = decoded[0][0]
self.assert_gts_acars_uk1_contents(msg)
def test_issue228(self):
# update from file
with self._explorer() as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/issue228.bufr")) as message:
updater.add_messages(message)
self.assertEqual(explorer.stats.datetime_min, None)
self.assertEqual(explorer.stats.datetime_max, None)
self.assertEqual(explorer.stats.count, 5)
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/issue228.bufr")) as message:
updater.add_messages(message)
self.assertEqual(explorer.stats.datetime_min, None)
self.assertEqual(explorer.stats.datetime_max, None)
self.assertEqual(explorer.stats.count, 10)
def do_qc(input_file, output_file, preserve):
importer = dballe.Importer("BUFR")
exporter = dballe.Exporter("BUFR")
with importer.from_file(input_file) as fp:
for msgs in fp:
for msg in msgs:
count_vars = 0
new_msg = dballe.Message("generic")
new_msg.set_named("year", msg.datetime.year)
new_msg.set_named("month", msg.datetime.month)
new_msg.set_named("day", msg.datetime.day)
new_msg.set_named("hour", msg.datetime.hour)
new_msg.set_named("minute", msg.datetime.minute)
new_msg.set_named("second", msg.datetime.second)
new_msg.set_named("rep_memo", msg.report)
new_msg.set_named("longitude", int(msg.coords[0] * 10 ** 5))
new_msg.set_named("latitude", int(msg.coords[1] * 10 ** 5))
if msg.ident:
new_msg.set_named("ident", msg.ident)
for data in msg.query_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
is_ok = pass_qc(attrs)
v = dballe.var(
data["variable"].code, data["variable"].get()
)
if not is_ok:
if preserve:
v.seta(dballe.var("B33007", 0))
else:
continue
new_msg.set(data["level"], data["trange"], v)
count_vars += 1
for data in msg.query_station_data({"query": "attrs"}):
variable = data["variable"]
attrs = variable.get_attrs()
v = dballe.var(
data["variable"].code, data["variable"].get()
)
for a in attrs:
v.seta(a)
new_msg.set(dballe.Level(), dballe.Trange(), v)
if count_vars > 0:
output_file.write(exporter.to_binary(new_msg))
def test_issue160(self):
importer = dballe.Importer("BUFR")
codes = []
with importer.from_file(test_pathname("bufr/issue160.bufr")) as f:
for msgs in f:
for msg in msgs:
for d in msg.query_station_data():
codes.append(d["var"])
self.assertCountEqual(
codes,
("B01019", "B07030", "B07031", "B01194", "B04001", "B04002",
"B04003", "B04004", "B04005", "B04006", "B05001", "B06001"))
def test_issue197(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
importer = dballe.Importer("BUFR")
msgs = importer.from_binary(binmsg)
with msgs[0].query_data() as m:
with self.assertRaises(RuntimeError) as e:
m.data
self.assertEqual(
str(e.exception),
"cannot access values on a cursor before or after iteration")
def test_issue213(self):
importer = dballe.Importer("BUFR")
count_msg = 0
count_data = 0
with importer.from_file(
test_pathname("bufr/generic-onlystation.bufr")) as f:
for msgs in f:
for msg in msgs:
count_msg += 1
for d in msg.query_data():
d["trange"]
count_data += 1
self.assertEqual(count_msg, 1)
self.assertEqual(count_data, 0)
def test_fromfile_shortcut_byteio(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
with open(pathname, "rb") as fd:
data = fd.read()
with io.BytesIO(data) as fd:
with importer.from_file(fd) as f:
decoded = list(f)
self.assertEqual(len(decoded), 1)
self.assertEqual(len(decoded[0]), 1)
msg = decoded[0][0]
self.assert_gts_acars_uk1_contents(msg)
def main(inputfiles, out):
importer = dballe.Importer("BUFR")
out.write('{"type":"FeatureCollection", "features":[')
for f in inputfiles:
with importer.from_file(f) as fp:
is_first = True
for msgs in fp:
for msg in msgs:
for cur in msg.query_data():
lev = cur["level"]
tr = cur["trange"]
if not is_first:
out.write(",")
else:
is_first = False
var = cur["variable"]
json.dump({
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [cur.enqd("lon"), cur.enqd("lat")],
},
"properties": {
"lon": cur.enqi("lon"),
"lat": cur.enqi("lat"),
"datetime": cur["datetime"].strftime("%Y-%m-%dT%H:%M:%SZ"),
"network": cur["report"],
"ident": cur["ident"],
"level_t1": lev.ltype1 if lev is not None else None,
"level_v1": lev.l1 if lev is not None else None,
"level_t2": lev.ltype2 if lev is not None else None,
"level_v2": lev.l2 if lev is not None else None,
"trange_pind": tr.pind if tr is not None else None,
"trange_p1": tr.p1 if tr is not None else None,
"trange_p2": tr.p2 if tr is not None else None,
"bcode": var.code,
"value": var.get(),
}
}, out)
out.write("]}")
def test_not_preserve(self):
with open("tests/001.bufr", "rb") as fpin:
with io.BytesIO() as fpout:
dba_qcfilter.cli.do_qc(fpin, fpout, False)
fpout.seek(0)
importer = dballe.Importer("BUFR")
with importer.from_file(fpout) as fp:
count = 0
for msgs in fp:
for msg in msgs:
for data in msg.query_data():
attrs = data["variable"].get_attrs()
# Attributes are removed after QC
self.assertEqual(len(attrs), 0)
count += 1
# There's only one valid BUFR
self.assertEqual(count, 1)
def test_preserve(self):
with open("tests/001.bufr", "rb") as fpin:
with io.BytesIO() as fpout:
dba_qcfilter.cli.do_qc(fpin, fpout, True)
fpout.seek(0)
importer = dballe.Importer("BUFR")
with importer.from_file(fpout) as fp:
count = 0
invalid_count = 0
valid_count = 0
for msgs in fp:
for msg in msgs:
for data in msg.query_data():
attrs = {
a.code: a.get()
for a in data["variable"].get_attrs()
}
if "B33007" in attrs:
# If B33007 is set, it must be the only one
self.assertEqual(len(attrs.keys()), 1)
# If B33007 is set, its value must be 0
self.assertEqual(attrs["B33007"], 0)
invalid_count += 1
else:
# If B33007 is not set, then the attributes
# list must be empty
self.assertEqual(len(attrs.keys()), 0)
valid_count += 1
count += 1
# All BUFR messages must be in the output file
self.assertEqual(count, 3)
# Two bufr messages are invalid
self.assertEqual(invalid_count, 2)
# There's only one valid BUFR
self.assertEqual(valid_count, 1)
def main(host, keepalive, port, topics, username, password, debug, infile):
mqttclient = mqtt.Client(userdata={
"debug": debug,
})
if username:
mqttclient.username_pw_set(username, password)
mqttclient.on_log = on_log
mqttclient.connect(host, port=port, keepalive=keepalive)
handle_signals(mqttclient)
mqttclient.loop_start()
importer = dballe.Importer("BUFR")
with importer.from_file(infile) as f:
for topic, payload, is_station in convert_to_mqtt(f):
for basetopic in topics:
t = basetopic + topic
publish_to_mqtt(mqttclient, t, payload, is_station)
mqttclient.loop_stop()