def testLoadFileOverwriteAttrs(self): with io.open(test_pathname("bufr/issue91-withoutB33196.bufr"), "rb") as fp: self.db.remove_all() self.db.load(fp, attrs=True, overwrite=True) with self.deprecated_on_db(): with self.db.query_data() as cur: rec = next(cur) context_id = rec["context_id"] var = rec["variable"] with self.deprecated_on_db(): a = self.db.attr_query_data(context_id) self.assertEqual(var.code, "B12101") self.assertEqual(var.enq(), 274.15) self.assertTrue("B33196" not in a) with io.open(test_pathname("bufr/issue91-withB33196.bufr"), "rb") as fp: self.db.load(fp, attrs=True, overwrite=True) with self.deprecated_on_db(): with self.db.query_data() as cur: rec = next(cur) context_id = rec["context_id"] var = rec["variable"] with self.deprecated_on_db(): a = self.db.attr_query_data(context_id) self.assertEqual(var.code, "B12101") self.assertEqual(var.enq(), 273.15) self.assertTrue("B33196" in a)
def testLoadFileOverwrite(self): with io.open(test_pathname("bufr/issue91-withoutB33196.bufr"), "rb") as fp: self.db.remove_all() self.db.load(fp, overwrite=True) with self.deprecated_on_db(): with self.db.query_data() as cur: rec = next(cur) context_id = rec["context_id"] var = rec["variable"] with self.deprecated_on_db(): self.db.attr_query_data( context_id ) # cannot verify the result, but expecting not to raise self.assertEqual(var.code, "B12101") self.assertEqual(var.enqd(), 274.15) with io.open(test_pathname("bufr/issue91-withB33196.bufr"), "rb") as fp: self.db.load(fp, overwrite=True) with self.deprecated_on_db(): with self.db.query_data() as cur: rec = next(cur) context_id = rec["context_id"] var = rec["variable"] with self.deprecated_on_db(): self.db.attr_query_data( context_id ) # cannot verify the result, but expecting not to raise) self.assertEqual(var.code, "B12101") self.assertEqual(var.enqd(), 273.15)
def test_issue216(self): """ Check that one can import messages with either all data, station data only, measured data only """ with self._explorer("e1") as explorer: with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/gts-acars1.bufr")) as messages: updater.add_messages(messages) self.assertEqual(explorer.all_varcodes, [ 'B01011', 'B02062', 'B02064', 'B04001', 'B04002', 'B04003', 'B04004', 'B04005', 'B04006', 'B05001', 'B06001', 'B07030', 'B08004', 'B11001', 'B11002', 'B12101', 'B13002' ]) with self._explorer("e2") as explorer: with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/gts-acars1.bufr")) as messages: updater.add_messages(messages, station_data=True, data=False) self.assertEqual(explorer.all_varcodes, [ 'B01011', 'B04001', 'B04002', 'B04003', 'B04004', 'B04005', 'B04006', 'B05001', 'B06001' ]) with self._explorer("e3") as explorer: with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/gts-acars1.bufr")) as messages: updater.add_messages(messages, station_data=False, data=True) self.assertEqual(explorer.all_varcodes, [ 'B02062', 'B02064', 'B07030', 'B08004', 'B11001', 'B11002', 'B12101', 'B13002' ]) with self._explorer("e4") as explorer: with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/gts-acars1.bufr")) as messages: updater.add_messages(messages, station_data=False, data=False) self.assertEqual(explorer.all_varcodes, [])
def test_create_from_msgs(self): importer = dballe.Importer("BUFR") explorer = self._explorer() with explorer.rebuild() as update: with importer.from_file(test_pathname("bufr/gts-acars-uk1.bufr")) as imp: update.add_messages(imp) explorer.set_filter({"level": dballe.Level(102, 6260000)}) self.assertCountEqual(explorer.all_stations, [ self._station("amdar", None, 48.90500, 10.63667, "EU3375"), ]) self.assertCountEqual(explorer.stations, [ self._station("amdar", None, 48.90500, 10.63667, "EU3375"), ]) self.assertEqual(explorer.all_reports, ["amdar"]) self.assertEqual(explorer.reports, ["amdar"]) self.assertEqual(explorer.all_levels, [dballe.Level(102, 6260000), None]) self.assertEqual(explorer.levels, [dballe.Level(102, 6260000)]) self.assertEqual(explorer.all_tranges, [dballe.Trange(254, 0, 0), None]) self.assertEqual(explorer.tranges, [dballe.Trange(254, 0, 0)]) self.assertCountEqual(explorer.all_varcodes, [ "B04001", "B04002", "B04003", "B04004", "B04005", "B01011", "B05001", "B06001", "B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002", ]) self.assertCountEqual(explorer.varcodes, [ "B01006", "B02061", "B02062", "B02064", "B07030", "B08004", "B11001", "B11002", "B12101", "B13002", ]) self.assertEqual(explorer.all_stats, dballe.ExplorerStats(( datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 18))) self.assertEqual(explorer.stats, dballe.ExplorerStats(( datetime.datetime(2009, 2, 24, 11, 31), datetime.datetime(2009, 2, 24, 11, 31), 10)))
def test_domain_errors(self): pathname = test_pathname("bufr/interpreted-range.bufr") with dballe.File(pathname) as f: binmsg = next(f) with self.assertRaises(ValueError): dballe.Importer("BUFR", domain_errors="throw") importer = dballe.Importer("BUFR", domain_errors="raise") with self.assertRaises(OverflowError): importer.from_binary(binmsg) importer = dballe.Importer("BUFR", domain_errors="unset") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertFalse(val.isset) importer = dballe.Importer("BUFR", domain_errors="clamp") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertEqual(val.enqd(), 327.66) importer = dballe.Importer("BUFR", domain_errors="tag") msgs = importer.from_binary(binmsg) val = msgs[0].get(dballe.Level(1), dballe.Trange(254, 0, 0), "B22043") self.assertEqual(val.enqd(), 327.66) a = val.enqa("B33192") self.assertTrue(a) self.assertEqual(a.enqi(), 0)
def test_import_message_sequence(self): importer = dballe.Importer("BUFR") with dballe.File(test_pathname("bufr/vad.bufr")) as fp: for binmsg in fp: self.db.import_messages(importer.from_binary(binmsg)) with self.deprecated_on_db(): self.assertEqual(self.db.query_data({}).remaining, 371)
def testLoadFileLike(self): with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp: s = io.BytesIO(fp.read()) self.db.remove_all() self.db.load(s) with self.deprecated_on_db(): self.assertTrue(self.db.query_data().remaining > 0)
def test_refcounts(self): pathname = test_pathname("bufr/gts-acars-uk1.bufr") importer = dballe.Importer("BUFR") self.assertEqual(sys.getrefcount(importer), 2) # importer, getrefcount with dballe.File(pathname) as f: self.assertEqual(sys.getrefcount(importer), 2) # importer, getrefcount self.assertEqual(sys.getrefcount(f), 3) # __enter__ result, f, getrefcount fimp = importer.from_file(f) self.assertEqual(sys.getrefcount(importer), 3) # importer, fimp, getrefcount self.assertEqual(sys.getrefcount(f), 4) # __enter__ result, f, fimp, getrefcount decoded = list(fimp) self.assertEqual(sys.getrefcount(importer), 3) # importer, fimp, getrefcount self.assertEqual(sys.getrefcount(f), 4) # __enter__ result, f, fimp, getrefcount self.assertEqual(sys.getrefcount(importer), 3) # importer, fimp, getrefcount self.assertEqual(sys.getrefcount(f), 3) # f, fimp, getrefcount self.assertEqual(len(decoded), 1)
def test_import_varlist(self): with self.transaction() as tr: tr.remove_all() importer = dballe.Importer("BUFR") with dballe.File(test_pathname("bufr/vad.bufr")) as fp: tr.import_messages(importer.from_file(fp), varlist="B11001,B11002") for cur in tr.query_data(): self.assertIn(cur["var"], ("B11001", "B11002"))
def test_default(self): pathname = test_pathname("bufr/gts-acars-uk1.bufr") with dballe.File(pathname) as f: binmsg = next(f) importer = dballe.Importer("BUFR") msg = importer.from_binary(binmsg) self.assertEqual(len(msg), 1) self.assert_gts_acars_uk1_contents(msg[0])
def test_issue228(self): # update from file with self._explorer() as explorer: with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/issue228.bufr")) as message: updater.add_messages(message) self.assertEqual(explorer.stats.datetime_min, None) self.assertEqual(explorer.stats.datetime_max, None) self.assertEqual(explorer.stats.count, 5) with explorer.update() as updater: importer = dballe.Importer("BUFR") with importer.from_file( test_pathname("bufr/issue228.bufr")) as message: updater.add_messages(message) self.assertEqual(explorer.stats.datetime_min, None) self.assertEqual(explorer.stats.datetime_max, None) self.assertEqual(explorer.stats.count, 10)
def test_fromfile_shortcut_pathname(self): pathname = test_pathname("bufr/gts-acars-uk1.bufr") importer = dballe.Importer("BUFR") with importer.from_file(pathname) as f: decoded = list(f) self.assertEqual(len(decoded), 1) self.assertEqual(len(decoded[0]), 1) msg = decoded[0][0] self.assert_gts_acars_uk1_contents(msg)
def testLoadFileWithAttrs(self): with io.open(test_pathname("bufr/issue91-withB33196.bufr"), "rb") as fp: self.db.remove_all() self.db.load(fp, attrs=True) with self.deprecated_on_db(): with self.db.query_data() as cur: rec = next(cur) context_id = rec["context_id"] with self.deprecated_on_db(): a = self.db.attr_query_data(context_id) self.assertTrue("B33196" in a)
def test_issue160(self): importer = dballe.Importer("BUFR") codes = [] with importer.from_file(test_pathname("bufr/issue160.bufr")) as f: for msgs in f: for msg in msgs: for d in msg.query_station_data(): codes.append(d["var"]) self.assertCountEqual( codes, ("B01019", "B07030", "B07031", "B01194", "B04001", "B04002", "B04003", "B04004", "B04005", "B04006", "B05001", "B06001"))
def test_issue232(self): explorer = dballe.DBExplorer() with explorer.update() as updater: with open(test_pathname("json/issue232.json"), "rt") as fd: updater.add_json(fd.read()) query = { 'datetimemin': datetime.datetime(2020, 1, 1, 1, 0), 'datetimemax': datetime.datetime(2020, 6, 1, 15, 13), 'rep_memo': 'simnpr', 'query': 'details' } for cur in explorer.query_summary_all(query): cur["yearmin"]
def test_issue213(self): importer = dballe.Importer("BUFR") count_msg = 0 count_data = 0 with importer.from_file( test_pathname("bufr/generic-onlystation.bufr")) as f: for msgs in f: for msg in msgs: count_msg += 1 for d in msg.query_data(): d["trange"] count_data += 1 self.assertEqual(count_msg, 1) self.assertEqual(count_data, 0)
def test_issue197(self): pathname = test_pathname("bufr/gts-acars-uk1.bufr") with dballe.File(pathname) as f: binmsg = next(f) importer = dballe.Importer("BUFR") msgs = importer.from_binary(binmsg) with msgs[0].query_data() as m: with self.assertRaises(RuntimeError) as e: m.data self.assertEqual( str(e.exception), "cannot access values on a cursor before or after iteration")
def test_fromfile_shortcut_byteio(self): pathname = test_pathname("bufr/gts-acars-uk1.bufr") importer = dballe.Importer("BUFR") with open(pathname, "rb") as fd: data = fd.read() with io.BytesIO(data) as fd: with importer.from_file(fd) as f: decoded = list(f) self.assertEqual(len(decoded), 1) self.assertEqual(len(decoded[0]), 1) msg = decoded[0][0] self.assert_gts_acars_uk1_contents(msg)
def testLoadAutodetect(self): # BUFR, autodetectable with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp), 25) # BUFR, not autodetectable with io.open(test_pathname("bufr/synop-groundtemp.bufr"), "rb") as fp: self.db.remove_all() with self.assertRaises(KeyError): self.db.load(fp) # CREX, autodetectable with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp), 1) # BUFR with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp, "BUFR"), 25) # BUFR with io.open(test_pathname("bufr/synop-groundtemp.bufr"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp, "BUFR"), 1) # CREX loaded as BUFR yields no results with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp, "BUFR"), 0) # CREX with io.open(test_pathname("crex/test-synop0.crex"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp, "CREX"), 1) # BUFR loaded as CREX yields no results with io.open(test_pathname("bufr/vad.bufr"), "rb") as fp: self.db.remove_all() self.assertEqual(self.db.load(fp, "CREX"), 0)
def testLoadFileno(self): class F(object): def __init__(self, path): self.path = path def read(*args): raise AttributeError() def fileno(self): return self.fp.fileno() def __enter__(self): self.fp = open(self.path, "r") return self def __exit__(self, type, value, traceback): self.fp.close() with F(test_pathname("bufr/vad.bufr")) as f: self.db.remove_all() self.db.load(f) with self.deprecated_on_db(): self.assertTrue(self.db.query_data().remaining > 0)
def setUp(self): self.pathname = test_pathname("bufr/gts-acars-uk1.bufr")
def test_issue235(self): with dballe.Explorer(test_pathname("json/issue235.json")) as explorer: for cur in explorer.query_summary({"var": "B12101"}): cur["datetimemax"]