def testLoadFileOverwriteAttrs(self):
with io.open(test_pathname("bufr/issue91-withoutB33196.bufr"),
"rb") as fp:
self.db.remove_all()
self.db.load(fp, attrs=True, overwrite=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
rec = next(cur)
context_id = rec["context_id"]
var = rec["variable"]
with self.deprecated_on_db():
a = self.db.attr_query_data(context_id)
self.assertEqual(var.code, "B12101")
self.assertEqual(var.enq(), 274.15)
self.assertTrue("B33196" not in a)
with io.open(test_pathname("bufr/issue91-withB33196.bufr"),
"rb") as fp:
self.db.load(fp, attrs=True, overwrite=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
rec = next(cur)
context_id = rec["context_id"]
var = rec["variable"]
with self.deprecated_on_db():
a = self.db.attr_query_data(context_id)
self.assertEqual(var.code, "B12101")
self.assertEqual(var.enq(), 273.15)
self.assertTrue("B33196" in a)
def testLoadFileOverwrite(self):
with io.open(test_pathname("bufr/issue91-withoutB33196.bufr"),
"rb") as fp:
self.db.remove_all()
self.db.load(fp, overwrite=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
rec = next(cur)
context_id = rec["context_id"]
var = rec["variable"]
with self.deprecated_on_db():
self.db.attr_query_data(
context_id
) # cannot verify the result, but expecting not to raise
self.assertEqual(var.code, "B12101")
self.assertEqual(var.enqd(), 274.15)
with io.open(test_pathname("bufr/issue91-withB33196.bufr"),
"rb") as fp:
self.db.load(fp, overwrite=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
rec = next(cur)
context_id = rec["context_id"]
var = rec["variable"]
with self.deprecated_on_db():
self.db.attr_query_data(
context_id
) # cannot verify the result, but expecting not to raise)
self.assertEqual(var.code, "B12101")
self.assertEqual(var.enqd(), 273.15)
def test_issue216(self):
"""
Check that one can import messages with either all data, station data
only, measured data only
"""
with self._explorer("e1") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages)
self.assertEqual(explorer.all_varcodes, [
'B01011', 'B02062', 'B02064', 'B04001', 'B04002', 'B04003',
'B04004', 'B04005', 'B04006', 'B05001', 'B06001', 'B07030',
'B08004', 'B11001', 'B11002', 'B12101', 'B13002'
])
with self._explorer("e2") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=True,
data=False)
self.assertEqual(explorer.all_varcodes, [
'B01011', 'B04001', 'B04002', 'B04003', 'B04004', 'B04005',
'B04006', 'B05001', 'B06001'
])
with self._explorer("e3") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=False,
data=True)
self.assertEqual(explorer.all_varcodes, [
'B02062', 'B02064', 'B07030', 'B08004', 'B11001', 'B11002',
'B12101', 'B13002'
])
with self._explorer("e4") as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/gts-acars1.bufr")) as messages:
updater.add_messages(messages,
station_data=False,
data=False)
self.assertEqual(explorer.all_varcodes, [])
def test_create_from_msgs(self):
importer = dballe.Importer("BUFR")
explorer = self._explorer()
with explorer.rebuild() as update:
with importer.from_file(test_pathname("bufr/gts-acars-uk1.bufr")) as imp:
update.add_messages(imp)
explorer.set_filter({"level": dballe.Level(102, 6260000)})
self.assertCountEqual(explorer.all_stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertCountEqual(explorer.stations, [
self._station("amdar", None, 48.90500, 10.63667, "EU3375"),
])
self.assertEqual(explorer.all_reports, ["amdar"])
self.assertEqual(explorer.reports, ["amdar"])
self.assertEqual(explorer.all_levels, [dballe.Level(102, 6260000), None])
self.assertEqual(explorer.levels, [dballe.Level(102, 6260000)])
self.assertEqual(explorer.all_tranges, [dballe.Trange(254, 0, 0), None])
self.assertEqual(explorer.tranges, [dballe.Trange(254, 0, 0)])
self.assertCountEqual(explorer.all_varcodes, [
"B04001", "B04002", "B04003", "B04004", "B04005", "B01011", "B05001", "B06001",
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertCountEqual(explorer.varcodes, [
"B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002",
])
self.assertEqual(explorer.all_stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 18)))
self.assertEqual(explorer.stats, dballe.ExplorerStats((
datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 10)))
def test_domain_errors(self):
pathname = test_pathname("bufr/interpreted-range.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
with self.assertRaises(ValueError):
dballe.Importer("BUFR", domain_errors="throw")
importer = dballe.Importer("BUFR", domain_errors="raise")
with self.assertRaises(OverflowError):
importer.from_binary(binmsg)
importer = dballe.Importer("BUFR", domain_errors="unset")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertFalse(val.isset)
importer = dballe.Importer("BUFR", domain_errors="clamp")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
importer = dballe.Importer("BUFR", domain_errors="tag")
msgs = importer.from_binary(binmsg)
val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043")
self.assertEqual(val.enqd(), 327.66)
a = val.enqa("B33192")
self.assertTrue(a)
self.assertEqual(a.enqi(), 0)
def test_import_message_sequence(self):
importer = dballe.Importer("BUFR")
with dballe.File(test_pathname("bufr/vad.bufr")) as fp:
for binmsg in fp:
self.db.import_messages(importer.from_binary(binmsg))
with self.deprecated_on_db():
self.assertEqual(self.db.query_data({}).remaining, 371)
def testLoadFileLike(self):
with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp:
s = io.BytesIO(fp.read())
self.db.remove_all()
self.db.load(s)
with self.deprecated_on_db():
self.assertTrue(self.db.query_data().remaining > 0)
def test_refcounts(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
self.assertEqual(sys.getrefcount(importer), 2) # importer, getrefcount
with dballe.File(pathname) as f:
self.assertEqual(sys.getrefcount(importer),
2) # importer, getrefcount
self.assertEqual(sys.getrefcount(f),
3) # __enter__ result, f, getrefcount
fimp = importer.from_file(f)
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f),
4) # __enter__ result, f, fimp, getrefcount
decoded = list(fimp)
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f),
4) # __enter__ result, f, fimp, getrefcount
self.assertEqual(sys.getrefcount(importer),
3) # importer, fimp, getrefcount
self.assertEqual(sys.getrefcount(f), 3) # f, fimp, getrefcount
self.assertEqual(len(decoded), 1)
def test_import_varlist(self):
with self.transaction() as tr:
tr.remove_all()
importer = dballe.Importer("BUFR")
with dballe.File(test_pathname("bufr/vad.bufr")) as fp:
tr.import_messages(importer.from_file(fp),
varlist="B11001,B11002")
for cur in tr.query_data():
self.assertIn(cur["var"], ("B11001", "B11002"))
def test_default(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
importer = dballe.Importer("BUFR")
msg = importer.from_binary(binmsg)
self.assertEqual(len(msg), 1)
self.assert_gts_acars_uk1_contents(msg[0])
def test_issue228(self):
# update from file
with self._explorer() as explorer:
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/issue228.bufr")) as message:
updater.add_messages(message)
self.assertEqual(explorer.stats.datetime_min, None)
self.assertEqual(explorer.stats.datetime_max, None)
self.assertEqual(explorer.stats.count, 5)
with explorer.update() as updater:
importer = dballe.Importer("BUFR")
with importer.from_file(
test_pathname("bufr/issue228.bufr")) as message:
updater.add_messages(message)
self.assertEqual(explorer.stats.datetime_min, None)
self.assertEqual(explorer.stats.datetime_max, None)
self.assertEqual(explorer.stats.count, 10)
def test_fromfile_shortcut_pathname(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
with importer.from_file(pathname) as f:
decoded = list(f)
self.assertEqual(len(decoded), 1)
self.assertEqual(len(decoded[0]), 1)
msg = decoded[0][0]
self.assert_gts_acars_uk1_contents(msg)
def testLoadFileWithAttrs(self):
with io.open(test_pathname("bufr/issue91-withB33196.bufr"),
"rb") as fp:
self.db.remove_all()
self.db.load(fp, attrs=True)
with self.deprecated_on_db():
with self.db.query_data() as cur:
rec = next(cur)
context_id = rec["context_id"]
with self.deprecated_on_db():
a = self.db.attr_query_data(context_id)
self.assertTrue("B33196" in a)
def test_issue160(self):
importer = dballe.Importer("BUFR")
codes = []
with importer.from_file(test_pathname("bufr/issue160.bufr")) as f:
for msgs in f:
for msg in msgs:
for d in msg.query_station_data():
codes.append(d["var"])
self.assertCountEqual(
codes,
("B01019", "B07030", "B07031", "B01194", "B04001", "B04002",
"B04003", "B04004", "B04005", "B04006", "B05001", "B06001"))
def test_issue232(self):
explorer = dballe.DBExplorer()
with explorer.update() as updater:
with open(test_pathname("json/issue232.json"), "rt") as fd:
updater.add_json(fd.read())
query = {
'datetimemin': datetime.datetime(2020, 1, 1, 1, 0),
'datetimemax': datetime.datetime(2020, 6, 1, 15, 13),
'rep_memo': 'simnpr',
'query': 'details'
}
for cur in explorer.query_summary_all(query):
cur["yearmin"]
def test_issue213(self):
importer = dballe.Importer("BUFR")
count_msg = 0
count_data = 0
with importer.from_file(
test_pathname("bufr/generic-onlystation.bufr")) as f:
for msgs in f:
for msg in msgs:
count_msg += 1
for d in msg.query_data():
d["trange"]
count_data += 1
self.assertEqual(count_msg, 1)
self.assertEqual(count_data, 0)
def test_issue197(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
with dballe.File(pathname) as f:
binmsg = next(f)
importer = dballe.Importer("BUFR")
msgs = importer.from_binary(binmsg)
with msgs[0].query_data() as m:
with self.assertRaises(RuntimeError) as e:
m.data
self.assertEqual(
str(e.exception),
"cannot access values on a cursor before or after iteration")
def test_fromfile_shortcut_byteio(self):
pathname = test_pathname("bufr/gts-acars-uk1.bufr")
importer = dballe.Importer("BUFR")
with open(pathname, "rb") as fd:
data = fd.read()
with io.BytesIO(data) as fd:
with importer.from_file(fd) as f:
decoded = list(f)
self.assertEqual(len(decoded), 1)
self.assertEqual(len(decoded[0]), 1)
msg = decoded[0][0]
self.assert_gts_acars_uk1_contents(msg)
def testLoadAutodetect(self):
# BUFR, autodetectable
with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp), 25)
# BUFR, not autodetectable
with io.open(test_pathname("bufr/synop-groundtemp.bufr"), "rb") as fp:
self.db.remove_all()
with self.assertRaises(KeyError):
self.db.load(fp)
# CREX, autodetectable
with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp), 1)
# BUFR
with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp, "BUFR"), 25)
# BUFR
with io.open(test_pathname("bufr/synop-groundtemp.bufr"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp, "BUFR"), 1)
# CREX loaded as BUFR yields no results
with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp, "BUFR"), 0)
# CREX
with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp, "CREX"), 1)
# BUFR loaded as CREX yields no results
with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp:
self.db.remove_all()
self.assertEqual(self.db.load(fp, "CREX"), 0)
def testLoadFileno(self):
class F(object):
def __init__(self, path):
self.path = path
def read(*args):
raise AttributeError()
def fileno(self):
return self.fp.fileno()
def __enter__(self):
self.fp = open(self.path, "r")
return self
def __exit__(self, type, value, traceback):
self.fp.close()
with F(test_pathname("bufr/vad.bufr")) as f:
self.db.remove_all()
self.db.load(f)
with self.deprecated_on_db():
self.assertTrue(self.db.query_data().remaining > 0)
def setUp(self):
self.pathname = test_pathname("bufr/gts-acars-uk1.bufr")
def test_issue235(self):
with dballe.Explorer(test_pathname("json/issue235.json")) as explorer:
for cur in explorer.query_summary({"var": "B12101"}):
cur["datetimemax"]
