def testReadWriteVariousTypes(self):
a=_pyhl.nodelist()
b=_pyhl.node(_pyhl.GROUP_ID,"/info")
a.addNode(b)
b=_pyhl.node(_pyhl.ATTRIBUTE_ID,"/info/xscale")
b.setScalarValue(-1,0.85,"double",-1)
a.addNode(b)
b=_pyhl.node(_pyhl.ATTRIBUTE_ID,"/info/yscale")
b.setScalarValue(-1,1.0,"float",-1)
a.addNode(b);
b=_pyhl.node(_pyhl.ATTRIBUTE_ID,"/info/xsize")
b.setScalarValue(-1, 240, "int", -1)
a.addNode(b)
b=_pyhl.node(_pyhl.ATTRIBUTE_ID,"/info/ysize")
b.setScalarValue(-1,480,"long",-1)
a.addNode(b)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
self.assertEqual("double", a.fetchNode("/info/xscale").format())
self.assertEqual("float", a.fetchNode("/info/yscale").format())
self.assertEqual("int", a.fetchNode("/info/xsize").format())
self.assertEqual("long", a.fetchNode("/info/ysize").format())
def readCT(filename):
ct, cp = _cartesian.new(), _cartesianparam.new()
nodelist = _pyhl.read_nodelist(filename)
for n in NODENAMES:
nodelist.selectNode(n)
nodelist.fetch()
ct.defaultParameter = "CT"
ct.projection = _projection.new("MSG", "no description",
nodelist.getNode("/PROJECTION").data())
cp.setData(nodelist.getNode("/CT").data())
ysize = nodelist.getNode("/CT/N_LINES").data()
xsize = nodelist.getNode("/CT/N_COLS").data()
ULx = nodelist.getNode("/XGEO_UP_LEFT").data()
ULy = nodelist.getNode("/YGEO_UP_LEFT").data()
LRx = nodelist.getNode("/XGEO_LOW_RIGHT").data()
LRy = nodelist.getNode("/YGEO_LOW_RIGHT").data()
yscale = (ULy - LRy) / ysize
xscale = (LRx - ULx) / xsize
xoffset, yoffset = xscale / 2, yscale / 2 # Offsets to adjust LL and UR corners
LLx = LRx - (xsize * xscale) - xoffset
LLy = ULy - (ysize * yscale) - yoffset
URx = LRx + xoffset
URy = ULy + yoffset
ct.areaextent = (LLx, LLy, URx, URy) # Differs ~5 cm from PPS and PyTROLL
ct.xscale, ct.yscale = xscale, yscale
cp.quantity = "CT"
cp.gain, cp.offset = 1.0, 0.0
cp.nodata, cp.undetect = -1.0, 0.0
ct.addParameter(cp)
return ct
def testReadErroneouslyNullterminatedString(self):
nodelist = _pyhl.read_nodelist(self.STRINGSFIXTURE)
node = nodelist.fetchNode("/erroneously_nullterminated")
self.assertEqual("this is a null terminated string", node.data())
self.assertEqual("this is a null terminated string\x00",
node.rawdata())
nodelist = None
def load_file(self, filename, quantity):
nodelist = _pyhl.read_nodelist(filename)
names = nodelist.getNodeNames()
index = 1
finished = False
vol = None
if "/Conventions" in names:
conv = nodelist.fetchNode("/Conventions").data()
if conv == "ODIM_H5/V2_0":
vol = _rave.volume()
if vol == None:
return None
vol.longitude = nodelist.fetchNode("/where/lon").data() * math.pi / 180.0
vol.latitude = nodelist.fetchNode("/where/lat").data() * math.pi / 180.0
vol.height = nodelist.fetchNode("/where/height").data()
while not finished:
scanname = "/dataset%d"%index
if scanname in names:
scan = self.load_scan_from_file(scanname, nodelist, names, quantity)
if scan != None:
vol.addScan(scan)
else:
finished = True
index = index + 1
#vol.sortByElevations(1) # Ascending sort
return vol
def testWriteNamedCompoundAttribute(self):
a=_pyhl.nodelist()
rinfo_obj =_rave_info_type.object()
rinfo_type=_rave_info_type.type()
rinfo_obj.xsize = 98
rinfo_obj.ysize = 97
rinfo_obj.xscale = 120.0
rinfo_obj.yscale = 130.0
rinfo_obj.area_extent = (33.0,32.0,31.0,30.0)
self.addTypeNode(a, "/RaveType", rinfo_type.hid())
self.addScalarValueNode(a, _pyhl.ATTRIBUTE_ID, "/attribute", rinfo_type.size(), rinfo_obj.tostring(), "compound", rinfo_type.hid())
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/RaveType")
self.assertEqual("UNDEFINED", b.format())
self.assertEqual(_pyhl.TYPE_ID, b.type())
b=a.fetchNode("/attribute")
self.assertEqual("compound", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
result = b.compound_data()
self.assertEqual(98, result['xsize'])
self.assertEqual(97, result['ysize'])
self.assertEqual(120, result['xscale'])
self.assertEqual(130, result['yscale'])
self.assertTrue(numpy.all([33.0,32.0,31.0,30.0]==result['area_extent']))
def read_dataset(self, index=1):
"""
Reads dataset with the given index into the existing RAVE object.
Arguments:
int index: the index of the dataset to read
Returns: nothing, the dataset is read into self.
"""
if index > self.eval('/what/sets'):
raise IndexError("Don't have that many datasets.")
a = _pyhl.read_nodelist(self.__file__)
counter = 1
for k, i in a.getNodeNames().items():
if i == _pyhl.DATASET_ID:
if index == counter:
none1, tag, none2 = k.split('/')
try:
check = self.data[tag] # if already there, do nothing
return
except KeyError:
a.selectNode(k)
a.fetch()
self.data[tag] = a.getNode(k).data()
self.info.find(
k[1:]).text = tag # self.put doesn't work
self.set(k[1:], 'type', 'dataset')
return
else:
counter += 1
def testReadWriteSameFile(self): a=_pyhl.nodelist() b=_pyhl.node(_pyhl.GROUP_ID, "/slask") a.addNode(b) a.write(self.TESTFILE) a = _pyhl.read_nodelist(self.TESTFILE) a.write(self.TESTFILE2)
def __init__(self, filename, quantities):
self._filename = filename
if not _pyhl.is_file_hdf5(filename):
raise Exception("Not a HDF5 file")
self._nodelist = _pyhl.read_nodelist(self._filename)
self._nodelist.selectAll()
self._nodelist.fetch()
self._nodenames = self._nodelist.getNodeNames().keys()
self._converted_files = [
] # Contains tuples of (nodelist, suggested name)
def open_hdf5(filename):
datadict = {}
a = _pyhl.read_nodelist(filename)
a.selectAll()
a.fetch()
node_names = a.getNodeNames()
items = []
for nodename in node_names.keys():
b = a.getNode(nodename)
items.append((nodename, b, b.type()))
items.sort() # guarantees groups before attributes
h5rad = rave_info.INFO("h5rad", version=H5RAD_VERSION)
groupmapping = {"" : h5rad}
for nodename, node, typ in items:
index = nodename.rindex("/")
parentname, tag = nodename[:index], nodename[index+1:]
# Deal with (ignore) H5IM stubs
#if tag in ["CLASS", "IMAGE_VERSION"]:
if os.path.split(parentname)[1] == 'data':
continue
e = SubElement(groupmapping[parentname], tag)
if typ==1:
groupmapping[nodename] = e # save the nodename to element mapping
elif typ==0:
t = h5type(node.data())
if t == "sequence":
# convert list to string
nodes = []
for n in node.data():
node = n.strip()
node = remove_nulls(str(node))
nodes.append(("'"+node+"'"))
e.text = ", ".join(nodes)
else:
e.text = remove_nulls(str(node.data()))
if t != "string":
e.attrib["type"] = t
elif typ==2:
datadict[nodename] = node.data()
e.attrib["type"] = "dataset"
e.text=nodename
## label = string.replace(parentname, "/", "")
## print parentname, label
## if label.startswith("profile"): # relic from 717 ...
## label = label + "_" + tag
## datadict[label] = node.data()
## e.attrib["type"] = "dataset"
## e.text=label
return h5rad, datadict, items
def testWriteInt(self):
a=_pyhl.nodelist()
self.addScalarValueNode(a, _pyhl.ATTRIBUTE_ID, "/intvalue", -1, -123, "int", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/intvalue")
self.assertEqual("int", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertEqual(-123, b.data())
def testWriteStringArray(self):
a=_pyhl.nodelist()
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/stringarray", -1, [4], ["abc", "def", "ghi", "jkl"], "string", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/stringarray")
self.assertEqual("string", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertTrue(numpy.all(["abc", "def", "ghi", "jkl"] == b.data()))
def testWriteLong(self):
a=_pyhl.nodelist()
self.addScalarValueNode(a, _pyhl.ATTRIBUTE_ID, "/longvalue", -1, 987654, "long", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/longvalue")
self.assertEqual("long", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertEqual(987654, b.data())
def testWriteDoubleArray(self):
a=_pyhl.nodelist()
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/doublearray", -1, [4], [1.1,2.2,3.3,4.4], "double", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/doublearray")
self.assertEqual("double", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertTrue(numpy.all([1.1,2.2,3.3,4.4] == b.data()))
def testWriteDouble(self):
a=_pyhl.nodelist()
self.addScalarValueNode(a, _pyhl.ATTRIBUTE_ID, "/doublevalue", -1, 1.123, "double", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/doublevalue")
self.assertEqual("double", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertAlmostEqual(1.123, b.data(), 4)
def testWriteLongArray(self):
a=_pyhl.nodelist()
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/longarray", -1, [4], [1,2,3,4], "long", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/longarray")
self.assertEqual("long", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertTrue(numpy.all([1,2,3,4] == b.data()))
def testWriteString(self):
#execute
a=_pyhl.nodelist()
self.addScalarValueNode(a, _pyhl.ATTRIBUTE_ID, "/stringvalue", -1, "My String", "string", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/stringvalue")
self.assertEqual("string", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertEqual("My String", b.data())
def testWriteReference(self):
a=_pyhl.nodelist()
self.addArrayValueNode(a, _pyhl.DATASET_ID, "/doublearray", -1, [4], [1.1,2.2,3.3,4.4], "double", -1)
self.addReference(a, "/reference", "/doublearray")
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/reference")
self.assertEqual("string", b.format())
self.assertEqual(_pyhl.REFERENCE_ID, b.type())
self.assertEqual("/doublearray", b.data())
def XtestVerifyImageAttributesWritten(self):
obj = rave.open(self.OLD_NRD_FORMAT_TESTFILE)
self.assertEquals(None, obj.get("/image1/data/CLASS"))
self.assertEquals(None, obj.get("/image1/data/IMAGE_VERSION"))
obj.save(self.TESTFILE)
# Verify
nodelist = _pyhl.read_nodelist(self.TESTFILE)
node = nodelist.fetchNode("/image1/data/CLASS")
self.assertEquals("IMAGE", node.data())
node = nodelist.fetchNode("/image1/data/IMAGE_VERSION")
self.assertEquals("1.2", node.data())
def readFile():
# There is no meaning creating the type
aList = _pyhl.read_nodelist("compound_test.hdf")
# Fetch the node
aNode = aList.fetchNode("/myCompoundAttribute")
# Translate from the string representation to object
cdescr = aNode.compound_data()
print "XSIZE="+`cdescr["xsize"]`
print "YSIZE="+`cdescr["ysize"]`
print "XSCALE="+`cdescr["xscale"]`
print "YSCALE="+`cdescr["yscale"]`
def readHac(self, fstr):
if os.path.isfile(fstr):
nodelist = _pyhl.read_nodelist(fstr)
nodelist.selectNode("/accumulation_count")
nodelist.selectNode("/hit_accum")
nodelist.fetch()
self.hac = _ravefield.new()
self.hac.addAttribute("how/count",
nodelist.getNode("/accumulation_count").data())
self.hac.setData(nodelist.getNode("/hit_accum").data())
else:
raise IOError, "No such HAC file: %s" % fstr
def testWriteGroup(self):
a=_pyhl.nodelist()
self.addGroupNode(a, "/group1")
self.addGroupNode(a, "/group1/group11")
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/group1")
self.assertEqual("UNDEFINED", b.format())
self.assertEqual(_pyhl.GROUP_ID, b.type())
b=a.fetchNode("/group1/group11")
self.assertEqual("UNDEFINED", b.format())
self.assertEqual(_pyhl.GROUP_ID, b.type())
def testWriteDoubleDataset(self):
a=_pyhl.nodelist()
c=numpy.arange(100)
c=numpy.array(c.astype(numpy.float64),numpy.float64)
c=numpy.reshape(c,(10,10)).astype(numpy.float64)
self.addArrayValueNode(a, _pyhl.DATASET_ID, "/doubledataset", -1, numpy.shape(c), c, "double", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/doubledataset")
self.assertEqual("double", b.format())
self.assertEqual(_pyhl.DATASET_ID, b.type())
self.assertTrue(numpy.all(c == b.data()))
def testWriteUcharDataset(self):
a=_pyhl.nodelist()
c=numpy.arange(100)
c=numpy.array(c.astype(numpy.uint8),numpy.uint8)
c=numpy.reshape(c,(10,10)).astype(numpy.uint8)
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/uchardataset", -1, numpy.shape(c), c, "uchar", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/uchardataset")
self.assertEqual("uchar", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
self.assertTrue(numpy.all(c == b.data()))
def testWriteFloatArray(self):
a=_pyhl.nodelist()
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/floatarray", -1, [4], [1.1,2.2,3.3,4.4], "float", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/floatarray")
self.assertEqual("float", b.format())
self.assertEqual(_pyhl.ATTRIBUTE_ID, b.type())
c = b.data()
self.assertEqual(4, len(c))
self.assertAlmostEqual(1.1, c[0], 4)
self.assertAlmostEqual(2.2, c[1], 4)
self.assertAlmostEqual(3.3, c[2], 4)
self.assertAlmostEqual(4.4, c[3], 4)
def get_lat_lon_1km_aggregated_mersi(satscene, options):
"""Read latitude and longitude for each (aggregated) pixel.
"""
# Example: FY3A_MERSI_GBAL_L1_20100308_0915_1000M_MS.HDF
filename = satscene.time_slot.strftime("FY3A_MERSI_GBAL_L1_%Y%M%D_%H%M_1000M_MS.HDF")
filename = os.path.join(options["dir"], filename)
a = _pyhl.read_nodelist(filename)
b = a.getNodeNames()
# Should only select/fetch the datasets needed. FIXME!
a.selectAll()
a.fetch()
lat = a.getNode("/Latitude").data()
lon = a.getNode("/Longitude").data()
return lat, lon
def testWriteLongLongArray(self):
a=_pyhl.nodelist()
try:
self.addArrayValueNode(a, _pyhl.ATTRIBUTE_ID, "/llongarray", -1, [4], [1,2,3,4], "llong", -1)
self.fail("Expected TypeError")
except TypeError:
pass
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
try:
a.fetchNode("/llongarray")
self.fail("Expected IOError")
except IOError:
pass
def testWriteLongLongDataset(self):
a=_pyhl.nodelist()
c=numpy.arange(100)
c=numpy.array(c.astype(numpy.int64),numpy.int64)
c=numpy.reshape(c,(10,10)).astype(numpy.int64)
self.addArrayValueNode(a, _pyhl.DATASET_ID, "/llongdataset", -1, numpy.shape(c), c, "llong", -1)
a.write(self.TESTFILE)
#verify
a=_pyhl.read_nodelist(self.TESTFILE)
b=a.fetchNode("/llongdataset")
if (_varioustests.sizeoflong() < _varioustests.sizeoflonglong()):
self.assertEqual("llong", b.format())
else:
self.assertEqual("long", b.format())
self.assertEqual(_pyhl.DATASET_ID, b.type())
self.assertTrue(numpy.all(c == b.data()))
def get_lat_lon_1km_aggregated_mersi(satscene, options):
"""Read latitude and longitude for each (aggregated) pixel.
"""
# Example: FY3A_MERSI_GBAL_L1_20100308_0915_1000M_MS.HDF
filename = satscene.time_slot.strftime(
"FY3A_MERSI_GBAL_L1_%Y%M%D_%H%M_1000M_MS.HDF")
filename = os.path.join(options["dir"], filename)
a = _pyhl.read_nodelist(filename)
b = a.getNodeNames()
# Should only select/fetch the datasets needed. FIXME!
a.selectAll()
a.fetch()
lat = a.getNode("/Latitude").data()
lon = a.getNode("/Longitude").data()
return lat, lon
def readFile():
aList = _pyhl.read_nodelist("simple_test.hdf")
# Select individual nodes, instead of all of them
aList.selectNode("/info/xscale")
aList.selectNode("/info/yscale")
aList.selectNode("/data")
# Fetch the data for selected nodes
aList.fetch()
# Print the data
aNode = aList.getNode("/info/xscale")
print "XSCALE=" + ` aNode.data() `
aNode = aList.getNode("/info/yscale")
print "YSCALE=" + ` aNode.data() `
aNode = aList.getNode("/data")
print "DATA=" + ` aNode.data() `
def get_metadata(filename):
a = _pyhl.read_nodelist(filename)
a.selectMetadata()
a.fetch()
node_names = a.getNodeNames()
items = []
for nodename in node_names.keys():
b = a.getNode(nodename)
items.append((nodename, b, b.type()))
items.sort() # guarantees groups before attributes
h5rad = rave_info.INFO("h5rad", version=H5RAD_VERSION)
groupmapping = {"" : h5rad}
for nodename, node, typ in items:
index = nodename.rindex("/")
parentname, tag = nodename[:index], nodename[index+1:]
e = SubElement(groupmapping[parentname], tag)
if typ==1:
groupmapping[nodename] = e # save the nodename to element mapping
elif typ==0:
t = h5type(node.data())
if t == "sequence":
# convert list to string
nodes = []
for n in node.data():
node = n.strip()
node = remove_nulls(str(node))
nodes.append(("'"+node+"'"))
e.text = ", ".join(nodes)
else:
e.text = remove_nulls(str(node.data()))
if t != "string":
e.attrib["type"] = t
# Skip typ==2, dataset array
# return only h5rad
return h5rad
def testFetch_from_dataset_2(self):
nodelist = _pyhl.read_nodelist(self.TESTFILE, "/dataset1/")
node = nodelist.fetchNode("/dataset1/attribute1")
self.assertEqual("/dataset1/attribute1", node.name())
node = nodelist.fetchNode("/dataset1/doublearray")
self.assertEqual("/dataset1/doublearray", node.name())
def load_1km_aggregated_mersi(satscene, options):
"""Read 1km agregated mersi data from file and load it into *satscene*.
"""
# Example: FY3A_MERSI_GBAL_L1_20100308_0915_1000M_MS.HDF
filename = satscene.time_slot.strftime("FY3A_MERSI_GBAL_L1_%Y%m%d_%H%M_1000M_MS.HDF")
filename = os.path.join(options["dir"], filename)
a = _pyhl.read_nodelist(filename)
b = a.getNodeNames()
# Should only select/fetch the datasets needed. FIXME!
a.selectAll()
a.fetch()
# MERSI Channel 1-4: EV_250_Aggr.1KM_RefSB
# MERSI Channel 5: EV_250_Aggr.1KM_Emissive
# MERSI Channel 6-20: EV_1KM_RefSB
datasets = ["/EV_250_Aggr.1KM_RefSB", "/EV_250_Aggr.1KM_Emissive", "/EV_1KM_RefSB"]
for nodename in datasets:
band_data = a.getNode(nodename).data()
valid_range = a.getNode("%s/valid_range" % (nodename)).data()
band_names = a.getNode("%s/band_name" % (nodename)).data().split(",")
# Special treatment of the situation where the bandnames are stored
# as '6~20' (quite inconvenient):
if "6~20" in band_names:
band_names = ["6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20"]
for (i, band) in enumerate(band_names):
if band not in satscene.channels_to_load:
continue
satscene[band] = np.ma.masked_outside(band_data[i], valid_range[0], valid_range[1], copy=False)
satscene[band].info = {
"var_name": "ch" + str(band),
"var_data": satscene[band].data,
"var_dim_names": ("x", "y"),
"_FillValue": 32767,
"standard_name": "",
"short_name": band,
"scale_factor": 1.0,
"add_offset": 0.0,
}
satscene.info = {
"var_children": [ # {'var_name' : 'lat', 'var_callback': Functor(satscene.get_lat, low_res), 'var_dim_names': ('x','y') },
# {'var_name' : 'lon', 'var_callback' : Functor(satscene.get_lon, low_res) , 'var_dim_names': ('x','y') },
## {'var_name' : 'lat_hrvis', 'var_data' : satscene.lat[high_res]},
## {'var_name' : 'lon_hrvis', 'var_data' : satscene.lon[high_res]},
],
"Satellite": satscene.fullname,
"Antenna": "None",
"Receiver": "SMHI",
"Time": satscene.time_slot.strftime("%Y-%m-%d %H:%M:%S UTC"),
#'Area_Name' : satscene.area_id,
"Area_Name": "swath",
"Projection": "satproj",
"Platform": "fengyun",
"Number": "3a",
"Service": "",
#'Columns' : satscene.channels[0].shape[1],
#'Lines' : satscene.channels[0].shape[0],
"SampleX": 1.0,
"SampleY": 1.0,
"title": "MERSI Level 1",
}
