def setUp(self):
self.db = db = DataBlock()
Sdims = (DimRep("freq", [1, 2]), DimRep("r", [1]), DimMatrix_i("i", 2),
DimMatrix_j("j", 2))
db.S = hfarray(np.array([[11, 12], [21, 22]])[None, None, :, :] *
np.array([[10], [20]])[..., None, None],
dims=Sdims)
db.V = hfarray([1.23], Sdims[1:2])
db.Y = hfarray([1.23], (DimRep("k", [1]), ))
def setUp(self):
d = DataBlock()
fi = DimSweep("Freq[Hz]", 3)
gi = DimRep("g", 2)
hi = DimRep("h", 3)
ji = DimRep("j", 4)
dims = (fi, )
d["a"] = hfarray([3, 2, 1], dims=dims, outputformat="%.5f", unit="V")
d["b"] = hfarray([30, 10], dims=(gi, ), outputformat="%.7f", unit="A")
d["c"] = hfarray([[30, 20, 10, 40], [20, 30, 40, 10]],
dims=(gi, hi),
outputformat="%.7f",
unit="A")
self.d = d
def test_1(self):
db = DataBlock()
Sdims = (DimRep("freq", [1, 2]), DimRep("r", [1]), DimMatrix_i("i", 2),
DimMatrix_j("j", 2))
db.S = hfarray(
np.array([[11, 12], [21, 22]])[np.newaxis, np.newaxis, :, :] *
np.array([[10], [20]])[..., np.newaxis, np.newaxis],
dims=Sdims)
db.V = hfarray([1.23], Sdims[1:2])
db.Y = hfarray([1.23], (DimRep("k", [1]), ))
out = dset.subset_datablock_by_dims(
dset.convert_matrices_to_elements(db), Sdims[:-2])
self.assertTrue("V" in out)
self.assertTrue("S11" in out)
self.assertTrue("S12" in out)
self.assertTrue("S21" in out)
self.assertTrue("S22" in out)
self.assertFalse("Y" in out)
def test_cant_append(self):
i1 = DimSweep("a", 1, unit="s", outputformat="")
i2 = DimSweep("INDEX", 2, unit="s")
iexpand = DimRep("INDEX", 1, unit="s", outputformat="")
d1 = DataBlock()
d1.b = hfarray([2], dims=(i1, ), outputformat="")
d1.c = hfarray(3)
d2 = DataBlock()
d2.b = hfarray([3], dims=(i1, ))
fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5"
with hdf5context(fname, mode="w") as fil:
savefun(d1, fil, expandable=True, expanddim=iexpand)
self.assertRaises(ValueError, append_hdf5, d2, fil)
fname.unlink()
def parse_blocks(self, stream):
for comments, header, data in stream:
db = DataBlock()
db.comments = Comments(comments)
header = header[0].strip().split("\t")
Nhead = len(header)
#data = np.array(data)
if Nhead != len(data[0]):
msg = "Different number of header variables "\
"from data columns"
raise SPDataIOError(msg)
output = DataDict()
for varname, column in zip(header, zip(*data)):
output.setdefault(varname, []).append(column)
for varname in output:
data = output[varname]
if len(data) > 1:
output[varname] = np.array(output[varname], order="F").T
else:
output[varname] = np.array(output[varname][0])
freq = DimSweep(header[0], output[header[0]])
db[header[0]] = freq
for x in output.keys()[1:]:
if output[x].ndim == 1:
db[x] = hfarray(output[x], dims=(freq, ))
else:
repdim = DimRep("rep", output[x].shape[1])
db[x] = hfarray(output[x], dims=(freq, repdim)).squeeze()
remove = []
for vname in db.comments.property:
if vname[:1] == "@":
unit = db.comments.property[vname].unit
data = [float(db.comments.property[vname])]
db[vname[1:]] = DimPartial(vname[1:], data=data, unit=unit)
remove.append(vname)
for v in remove:
del db.comments.property[v]
db.comments.fullcomments = [
com for com in db.comments.fullcomments
if not com.startswith("@")
]
yield db
def merge_variable(association_list):
names, indices, association_list =\
association_list_in_hypercube_order(association_list)
innershape = tuple(len(x) for x in indices)
innerdims = tuple(DimRep(name, i) for name, i in zip(names, indices))
if len(association_list) == 1:
#pdb.set_trace()
return association_list[0][1], {}, None
basedims = association_list[0][1].dims
prototype = association_list[0][1]
if prototype.dtype.type in (np.unicode_, np.str_):
length = max(int(x[1].dtype.str[2:]) for x in association_list)
proto_dtype = np.dtype((prototype.dtype.type, length))
else:
proto_dtype = prototype.dtype
newshape = prototype.shape[:1] + innershape + prototype.shape[1:]
result = empty(newshape, dtype=proto_dtype)
unit = []
for idx, x in association_list:
unit.append(getattr(x, "unit", None))
if prototype.ndim:
result[(slice(None, None, None), ) + idx] = x
else:
if x.ndim == 0:
result[idx] = x.flatten()[0]
else:
result[idx] = x
if len(set(unit)) == 1:
unit = unit.pop()
else:
unit = None
v = hfarray(result,
dims=basedims[:1] + innerdims + basedims[1:],
unit=unit)
if len(association_list[0][0]) == 0:
indexvars = {}
else:
indexvars = dict((x.name, hfarray(x)) for x in innerdims)
return v, indexvars, innerdims
def save_hdf5_handle(db, filehandle, expandable=False, expanddim=None, **kw):
if expanddim is None:
expanddim = DimRep("INDEX", 1)
filehandle.attrs["hftools file version"] = "0.2"
for k, v in db.ivardata.items():
ek = escape_varname(k)
create_dataset(filehandle, ek, v.data)
filehandle[ek].attrs["dimtype"] = v.__class__.__name__
if v.unit:
filehandle[ek].attrs["unit"] = v.unit
if v.outputformat:
filehandle[ek].attrs["outputformat"] = v.outputformat
if expandable:
data = np.array(expanddim.data)
filehandle.create_dataset(expanddim.name, data=data, maxshape=(None, ))
filehandle[expanddim.name].attrs["dtype"] = data.dtype.str
expanddim_class = expanddim.__class__.__name__
filehandle[expanddim.name].attrs["dimtype"] = expanddim_class
if expanddim.unit:
filehandle[expanddim.name].attrs["unit"] = expanddim.unit
if expanddim.outputformat:
outputformat = expanddim.outputformat
filehandle[expanddim.name].attrs["outputformat"] = outputformat
for k, v in db.vardata.items():
ek = escape_varname(k)
if expandable and v.ndim == 0:
v = v.add_dim(expanddim)
elif expandable:
v = v.add_dim(expanddim, 1)
else:
pass
create_dataset(filehandle, ek, v, expandable=expandable)
for idx, dv in enumerate(v.dims):
filehandle[ek].dims.create_scale(filehandle[dv.name])
filehandle[ek].dims[idx].attach_scale(filehandle[dv.name])
filehandle[ek].attrs["arraytype"] = v.__class__.__name__
if v.unit:
filehandle[ek].attrs["unit"] = v.unit
if v.outputformat:
filehandle[ek].attrs["outputformat"] = v.outputformat
def setUp(self):
d = DataBlock()
gi = DimSweep("Index", 6)
ri = DimRep("Rep", 2)
dims = (gi, )
d["a"] = hfarray([1, 1, 2, 2, 3, 3],
dims=dims,
outputformat="%.5f",
unit="V")
d["b"] = hfarray([10, 20, 10, 20, 10, 20],
dims=(gi, ),
outputformat="%.7f",
unit="A")
d["c"] = hfarray([10.1, 20.1, 10.2, 20.2, 10.3, 20.3],
dims=(gi, ),
outputformat="%.7f",
unit="A")
d["d"] = hfarray([17, 19], dims=(ri, ), outputformat="%.7f", unit="A")
self.d = d
def append_hdf5(db, filehandle, expanddim=None, **kw):
key = "hftools file version"
if key not in filehandle.attrs or filehandle.attrs[key] != "0.2":
raise Exception("Can only append to hftools version 0.2")
if expanddim is None:
expanddim = DimRep("INDEX", 1)
idx = filehandle[expanddim.name][-1] + 1
expand_dataset(filehandle[expanddim.name], 1, 0)
filehandle[expanddim.name].value[-1] = idx
for k, v in filehandle.items():
if k in db.ivardata or k == expanddim.name:
continue
if k not in db.vardata:
raise ValueError("Variable %r missing in db, can not append" % k)
diskdata = filehandle[k]
ax = diskdata.maxshape.index(None)
expand_dataset(diskdata, 1, ax)
s = (slice(None),) * ax + (diskdata.shape[ax] - 1,)
diskdata[s] = db[k]
def simple_merge_blocks(blocks):
if len(blocks) == 1:
return blocks[0]
ri = DimRep("INDEX", len(blocks))
out = DataBlock()
#import pdb;pdb.set_trace()
for k in blocks[0].vardata:
out[k] = hfarray([b[k] for b in blocks],
dims=(ri, ) + blocks[0][k].dims)
if blocks[0][k].dims:
out[k] = out[k].reorder_dimensions(blocks[0][k].dims[0])
for k in blocks[0].ivardata:
if isinstance(blocks[0].ivardata[k], DimPartial):
out[k] = hfarray([b.ivardata[k].data[0] for b in blocks],
dims=(ri, ))
c = []
for b in blocks:
if b.comments is not None:
c.extend(b.comments.fullcomments)
out.comments = Comments()
out.comments.fullcomments = c
return out
def test_2(self):
i1 = DimSweep("a", 1, unit="s", outputformat="")
i2 = DimSweep("INDEX", 2, unit="s")
iexpand = DimRep("INDEX", 1, unit="s", outputformat="")
d1 = DataBlock()
d1.b = hfarray([2], dims=(i1, ), outputformat="")
d1.c = hfarray(3)
d2 = DataBlock()
d2.b = hfarray([3], dims=(i1, ))
d2.c = hfarray(2)
fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5"
with hdf5context(fname, mode="w") as fil:
savefun(d1, fil, expandable=True, expanddim=iexpand)
append_hdf5(d2, fil, expanddim=iexpand)
d = readfun(fname)
self.assertAllclose(hfarray([[2, 3]], dims=(i1, i2)), d.b)
self.assertAllclose(hfarray([3, 2], dims=(i2, )), d.c)
self.assertEqual(d.ivardata["a"].unit, "s")
self.assertEqual(d.ivardata["INDEX"].unit, "s")
self.assertEqual(d.ivardata["a"].outputformat, "%d")
self.assertEqual(d.ivardata["INDEX"].outputformat, "%d")
self.assertEqual(d.b.outputformat, "%d")
fname.unlink()
def test_2(self):
d = DataBlock()
d.FILENAME = hfarray(["foo.txt", "bar.txt"],
dims=(DimRep("FILEINDEX", [0, 1]), ))
d.blockname = "baz.txt"
self.assertEqual(d.blockname, "baz.txt")
