def merge_blocks(blocks, hyper=False, indexed=False):
db = DataBlock()
dimpartialgroups = OrderedDict()
for b in blocks:
parts = get_partials(b)
partgroup = dimpartialgroups.setdefault(tuple(parts.keys()), {})
for k, v in parts.items():
partgroup.setdefault(k, []).extend(v)
for idx, dims in enumerate(dimpartialgroups.values(), 1):
for k, v in dims.items():
dims = (DimSweep("INDEX%s" % idx, len(v)),)
db[k] = hfarray([x.data[0] for x in v],
dims=dims, unit=v[0].unit)
varnames = set()
for b in blocks:
for k, v in b.ivardata.items():
if k not in db:
db[k] = v
for k in b.vardata.keys():
varnames.add(k)
for vname in varnames:
v = []
for b in blocks:
if vname not in b:
continue
partials = get_partials(b)
v.append(b[vname])
if v:
k = tuple(partials.keys())
if k:
ri = (db[tuple(dimpartialgroups[k].keys())[0]].dims[0],)
else:
ri = tuple()
value = hfarray(v, dims=ri + v[0].dims, unit=v[0].unit)
if v[0].dims and isinstance(v[0].dims[0], DimSweep):
value = value.reorder_dimensions(v[0].dims[0])
db[vname] = value
cmt = Comments()
for block in blocks:
if block.comments:
cmt.extend(block.comments)
db.comments = cmt
db.blockname = blocks[0].blockname
if hyper:
for vnames in dimpartialgroups.keys():
if vnames:
hyperindex = db[vnames[0]].dims[0]
db = db.hyper(vnames, hyperindex, all=True)
db = db.squeeze()
return db
def merge_blocks(blocks, hyper=False, indexed=False):
db = DataBlock()
dimpartialgroups = OrderedDict()
for b in blocks:
parts = get_partials(b)
partgroup = dimpartialgroups.setdefault(tuple(parts.keys()), {})
for k, v in parts.items():
partgroup.setdefault(k, []).extend(v)
for idx, dims in enumerate(dimpartialgroups.values(), 1):
for k, v in dims.items():
dims = (DimSweep("INDEX%s" % idx, len(v)), )
db[k] = hfarray([x.data[0] for x in v], dims=dims, unit=v[0].unit)
varnames = set()
for b in blocks:
for k, v in b.ivardata.items():
if k not in db:
db[k] = v
for k in b.vardata.keys():
varnames.add(k)
for vname in varnames:
v = []
for b in blocks:
if vname not in b:
continue
partials = get_partials(b)
v.append(b[vname])
if v:
k = tuple(partials.keys())
if k:
ri = (db[tuple(dimpartialgroups[k].keys())[0]].dims[0], )
else:
ri = tuple()
value = hfarray(v, dims=ri + v[0].dims, unit=v[0].unit)
if v[0].dims and isinstance(v[0].dims[0], DimSweep):
value = value.reorder_dimensions(v[0].dims[0])
db[vname] = value
cmt = Comments()
for block in blocks:
if block.comments:
cmt.extend(block.comments)
db.comments = cmt
db.blockname = blocks[0].blockname
if hyper:
for vnames in dimpartialgroups.keys():
if vnames:
hyperindex = db[vnames[0]].dims[0]
db = db.hyper(vnames, hyperindex, all=True)
db = db.squeeze()
return db
def read_hdf5(h5file, name="datablock", **kw):
if isinstance(h5file, string_types):
fil = h5py.File(h5file, "r")
else:
fil = h5file
db = DataBlock()
grp = fil[name]
blockname = grp.attrs["Blockname"]
if blockname.lower() == "none":
blockname = None
db.blockname = blockname
comments = grp["Comments"]
if "fullcomments" in comments and len(comments["fullcomments"]):
db.comments = Comments([
cast_unicode(x).strip() for x in np.array(comments["fullcomments"])
])
else:
db.comments = Comments()
ivardata = grp["ivardata"]
vardata = grp["vardata"]
for k in ivardata:
v = ivardata[k]
datadtype = v.attrs[r"info\dtype"] or None
dimcls = dims_dict.get(v.attrs[r"info\class"], DimRep)
unit = str(v.attrs.get(r"info\unit", "none"))
if unit.lower() == "none":
unit = None
vdata = np.array(np.array(v), dtype=datadtype)
dim = dimcls(k, vdata, unit=unit)
db[k] = dim
for k in vardata:
v = vardata[k]
datadtype = v.attrs[r"data\dtype"] or None
dims = tuple(db.ivardata[cast_unicode(dimname)]
for dimname in v.attrs[r"info\name"])
unit = cast_unicode(v.attrs.get(r"data\unit", "none"))
if unit.lower() == "none":
unit = None
db[k] = hfarray(np.array(v), dtype=datadtype, dims=dims, unit=unit)
if isinstance(h5file, string_types):
fil.close()
if kw.get("property_to_vars", False):
db.values_from_property()
return db
def test_4(self):
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dim = DimSweep("f", 3, outputformat="%.1f")
d.freq = dim
fname = testpath / "testdata/hdf5/v02/savetest/res_4.hdf5"
self.savefun[0](d, fname)
fname.unlink()
def test_2(self):
d = DataBlock()
d.comments = Comments(["Hej=10"])
# import pdb;pdb.set_trace()
d.b = hfarray([2], dims=(DimSweep("a", 1), ))
fname = testpath / "testdata/hdf5/v02/savetest/res_2.hdf5"
self.savefun[0](d, fname)
fname.unlink()
def test_9(self):
d = DataBlock()
d.comments = Comments(["Hej=10"])
d.b = hfarray([2], dims=(DimSweep("a", 1), ), unit="V")
fname = testpath / "testdata/hdf5/v01/savetest/res_9.hdf5"
self.savefun[0](d, fname)
d2 = readfun(fname)
self.assertEqual(d2.b.unit, "V")
fname.unlink()
def merge_blocks_do_hyper(blocks):
#import pdb;pdb.set_trace()
outdata = DataBlock()
data = merge_blocks_to_association_list(blocks)
ivars = {}
for b in blocks:
for vname, v in b.ivardata.items():
if vname not in ivars:
ivars[vname] = v
free_vars = set()
for vname, assoc in data.items():
free_vars.add(zip(*assoc)[0])
free_vars = list(free_vars)
for vname, assoc in data.items():
v, indexvars, dim = merge_variable(assoc)
outdata[vname] = v
for iname, value in indexvars.items():
outdata.ivardata[iname] = value.dims[0]
for v in outdata.vardata.keys():
if v in outdata.ivardata:
del outdata.vardata[v]
for v in outdata.vardata.values():
for k in v.dims:
if k.name in outdata.ivardata:
if ((isinstance(k, DimRep) and
not isinstance(outdata.ivardata[k.name], DimRep))):
outdata.replace_dim(outdata.ivardata[k.name], k)
cmt = Comments()
for block in blocks:
if block.comments:
cmt.extend(block.comments)
outdata.comments = cmt
outdata.blockname = blocks[0].blockname
for vname, v in ivars.items():
if vname not in outdata:
outdata[vname] = v
#import pdb;pdb.set_trace()
return outdata
def merge_blocks_do_hyper(blocks):
#import pdb;pdb.set_trace()
outdata = DataBlock()
data = merge_blocks_to_association_list(blocks)
ivars = {}
for b in blocks:
for vname, v in b.ivardata.items():
if vname not in ivars:
ivars[vname] = v
free_vars = set()
for vname, assoc in data.items():
free_vars.add(zip(*assoc)[0])
free_vars = list(free_vars)
for vname, assoc in data.items():
v, indexvars, dim = merge_variable(assoc)
outdata[vname] = v
for iname, value in indexvars.items():
outdata.ivardata[iname] = value.dims[0]
for v in outdata.vardata.keys():
if v in outdata.ivardata:
del outdata.vardata[v]
for v in outdata.vardata.values():
for k in v.dims:
if k.name in outdata.ivardata:
if ((isinstance(k, DimRep)
and not isinstance(outdata.ivardata[k.name], DimRep))):
outdata.replace_dim(outdata.ivardata[k.name], k)
cmt = Comments()
for block in blocks:
if block.comments:
cmt.extend(block.comments)
outdata.comments = cmt
outdata.blockname = blocks[0].blockname
for vname, v in ivars.items():
if vname not in outdata:
outdata[vname] = v
#import pdb;pdb.set_trace()
return outdata
def test_5(self):
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dim = DimSweep("f", 3, outputformat="%.1f")
d.freq = dim
d.date = hfarray("2012-08-13 08:03:01", dtype="datetime64[us]")
fname = testpath / "testdata/hdf5/v02/savetest/res_5.hdf5"
self.savefun[0](d, fname)
d2 = readfun(fname)
fname.unlink()
self.assertEqual(d2.date, d.date)
def test_save_2(self):
d = hftools.file_formats.read_citi(testpath /
"testdata/citi/test1.citi")
d.comments = Comments(["klasjd:1"])
d.comments.fullcomments = ["kljlk"]
fname = testpath / "testdata/citi/savetest/test1.citi"
hftools.file_formats.save_citi(d, fname)
e = hftools.file_formats.read_citi(fname)
self.assertAllclose(d.S, e.S)
fname.unlink()
def test_10(self):
d = DataBlock()
d.comments = Comments(["Hej=10"])
# import pdb;pdb.set_trace()
d.b = hfarray([2], dims=(DimSweep("a", 1), ), unit="V")
d.b.outputformat = ""
fname = testpath / "testdata/hdf5/v02/savetest/res_10.hdf5"
self.savefun[0](d, fname)
d2 = readfun(fname)
self.assertEqual(d2.b.outputformat, "%d")
fname.unlink()
def test_11(self):
ia = DimSweep("a", 1)
d = DataBlock()
d.comments = Comments(["Hej=10"])
# import pdb;pdb.set_trace()
d.b = hfarray([u"kalle"], dims=(ia, ), unit="V")
d.c = hfarray(u"kalle")
fname = testpath / "testdata/hdf5/v02/savetest/res_11.hdf5"
self.savefun[0](d, fname)
d2 = readfun(fname)
self.assertEqual(d2.b[0], u"kalle")
self.assertEqual(d2.c, u"kalle")
fname.unlink()
def test_2(self):
d = DataBlock()
d.comments = Comments(["Hej=10"])
# import pdb;pdb.set_trace()
d.b = hfarray([2], dims=(DimSweep("a", 1), ))
f1 = testpath / "testdata/sp-data/savetest/res_2.txt"
f2 = testpath / "testdata/sp-data/savetest/facit_2.txt"
hftools.file_formats.spdata.save_spdata(d, f1)
with open(f1) as resfil:
with open(f2) as facitfil:
for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)):
msg = "\nFailed on line %d\n result: %r\n facit: %r"
args = (idx + 1, rad1, rad2)
self.assertEqual(rad1, rad2, msg=msg % args)
def test_4(self):
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dim = DimSweep("f", 3, outputformat="%.1f")
d.freq = dim
fname = testpath / "testdata/sp-data/savetest/res_4.txt"
facitfname = testpath / "testdata/sp-data/savetest/facit_4.txt"
hftools.file_formats.spdata.save_spdata(d, fname)
with open(fname) as resfil:
with open(facitfname) as facitfil:
for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)):
msg = "\nFailed on line %d\n result: %r\n facit: %r"
args = (idx + 1, rad1, rad2)
self.assertEqual(rad1, rad2, msg=msg % args)
def parse_blocks(self, stream):
# raise Exception("HANTERAR INTE KOMMENTARER INNE I MANYOPTIONAL")
comments = []
comments.extend(ManyOptional("COMMENT")(stream))
citifileversion = Optional("CITIFILE")(stream)
comments.extend(ManyOptional("COMMENT")(stream))
name = One("NAME", "Must have END after DATA")(stream)
comments.extend(ManyOptional("COMMENT")(stream))
varnames = ManyOptional("VARDEF")(stream)
comments.extend(ManyOptional("COMMENT")(stream))
datanames = ManyOptional("DATADEF")(stream)
comments.extend(ManyOptional("COMMENT")(stream))
block = DataBlock()
block.blockname = name[0]
for name, typ, N in varnames:
N = int(N)
errmsg = "Missing VAR_LIST_BEGIN or SEG_LIST_BEGIN"
tagname, = OneOf(["VAR_LIST_BEGIN", "SEG_LIST_BEGIN"],
errmsg)(stream)
if tagname.startswith("VAR_LIST"):
datalist = handle_data(ManyOptional("DATA")(stream), typ)
block[name] = DimSweep(name, datalist)
One("VAR_LIST_END", "Missing VAR_LIST_END")(stream)
elif tagname.startswith("SEG"): # pragma: no branch
datalist, = One("SEG", "Missing SEG")(stream)
_, start, stop, step = datalist.strip().split()
block[name] = DimSweep(name,
np.linspace(float(start),
float(stop),
int(step)))
One("SEG_LIST_END", "Missing SEG_LIST_END")(stream)
else: # pragma: no cover
pass
comments.extend(ManyOptional("COMMENT")(stream))
for idx, com in enumerate(comments):
if com.startswith("CONSTANT TIME"):
date = tuple(com.strip().split()[2:])
comments[idx] = "Measurement time: %s-%s-%s %s:%s:%s" % date
block.comments = Comments(comments)
shape = tuple(block.ivardata[i[0]].data.shape[0] for i in varnames)
dims = tuple(block.ivardata[i[0]] for i in varnames)
for name, typ in datanames:
One("DATA_LIST_BEGIN", "Missing BEGIN")(stream)
datalist = array(handle_data(ManyOptional("DATA")(stream), typ))
datalist.shape = shape
block[name] = hfarray(datalist, dims=dims)
One("DATA_LIST_END", "Missing END")(stream)
yield block
def test_3(self):
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dims = (
DimSweep("f", 3, outputformat="%.1f"),
DimSweep("i", 2, outputformat="%.0f"),
DimSweep("j", 2, outputformat="%.0f"),
)
d.c = hfarray(
[[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]],
dims=dims,
outputformat="%.2f")
fname = testpath / "testdata/hdf5/v02/savetest/res_3.hdf5"
self.savefun[0](d, fname)
fname.unlink()
def parse_blocks(self, stream):
for comments, header, data in stream:
db = DataBlock()
db.comments = Comments(comments)
header = header[0].strip().split("\t")
Nhead = len(header)
#data = np.array(data)
if Nhead != len(data[0]):
msg = "Different number of header variables "\
"from data columns"
raise SPDataIOError(msg)
output = DataDict()
for varname, column in zip(header, zip(*data)):
output.setdefault(varname, []).append(column)
for varname in output:
data = output[varname]
if len(data) > 1:
output[varname] = np.array(output[varname], order="F").T
else:
output[varname] = np.array(output[varname][0])
freq = DimSweep(header[0], output[header[0]])
db[header[0]] = freq
for x in output.keys()[1:]:
if output[x].ndim == 1:
db[x] = hfarray(output[x], dims=(freq, ))
else:
repdim = DimRep("rep", output[x].shape[1])
db[x] = hfarray(output[x], dims=(freq, repdim)).squeeze()
remove = []
for vname in db.comments.property:
if vname[:1] == "@":
unit = db.comments.property[vname].unit
data = [float(db.comments.property[vname])]
db[vname[1:]] = DimPartial(vname[1:], data=data, unit=unit)
remove.append(vname)
for v in remove:
del db.comments.property[v]
db.comments.fullcomments = [
com for com in db.comments.fullcomments
if not com.startswith("@")
]
yield db
def parse_blocks(self, stream):
for comments, vars, blockname, header, attribs, data in stream:
db = DataBlock()
db.blockname = blockname = blockname[0]
#self.header = header = header[0]
db.comments = Comments(comments)
#data = numpy.array(data)
dd = self.proc_data(header, data[:])
for vname in dd:
db[vname] = dd[vname]
for var in vars:
vi = self.proc_var(var)
db[vi.name] = vi
for var in attribs:
name, value = self.proc_attrib(var)
db[name] = value
self.make_matrices(db, sum(header, []))
yield db
def test_3(self):
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dims = (
DimSweep("f", 3, outputformat="%.1f"),
DimSweep("i", 2, outputformat="%.0f"),
DimSweep("j", 2, outputformat="%.0f"),
)
d.c = hfarray(
[[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]],
dims=dims,
outputformat="%.2f")
f1 = testpath / "testdata/sp-data/savetest/res_3.txt"
f2 = testpath / "testdata/sp-data/savetest/facit_3.txt"
hftools.file_formats.spdata.save_spdata(d, f1)
with open(f1) as resfil:
with open(f2) as facitfil:
for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)):
msg = "\nFailed on line %d\n result: %r\n facit: %r"
args = (idx + 1, rad1, rad2)
self.assertEqual(rad1, rad2, msg=msg % args)
def simple_merge_blocks(blocks):
if len(blocks) == 1:
return blocks[0]
ri = DimRep("INDEX", len(blocks))
out = DataBlock()
#import pdb;pdb.set_trace()
for k in blocks[0].vardata:
out[k] = hfarray([b[k] for b in blocks],
dims=(ri, ) + blocks[0][k].dims)
if blocks[0][k].dims:
out[k] = out[k].reorder_dimensions(blocks[0][k].dims[0])
for k in blocks[0].ivardata:
if isinstance(blocks[0].ivardata[k], DimPartial):
out[k] = hfarray([b.ivardata[k].data[0] for b in blocks],
dims=(ri, ))
c = []
for b in blocks:
if b.comments is not None:
c.extend(b.comments.fullcomments)
out.comments = Comments()
out.comments.fullcomments = c
return out
def parse_blocks(self, stream):
comments = []
info = None
datalist = []
for token, lineno, rad in stream:
if token == "Comments":
comments.append(rad)
elif token == "Info":
if info is None:
info = rad
else:
msg = "Second # info at lineno: %d" % lineno
raise TouchstoneError(msg)
else:
datalist.append(list(map(float, rad.split())))
if info is None:
raise TouchstoneError("No # info line in file")
comments = Comments(comments)
f, data, fn, noisedata = proc_data(datalist)
out = proc_info(info, f, data, fn, noisedata)
#HZ S RI R 50
#comments.add_from_comment("!INFO:#%s"%(" ".join(info)))
out.comments = comments
yield out
make_complex=make_complex,
property_to_vars=property_to_vars,
guess_unit=guess_unit,
normalize=normalize,
make_matrix=make_matrix,
merge=merge,
verbose=verbose)
return res
if __name__ == "__main__":
a = read_touchstone("tests/testdata/touchstone/test4.s2p",
make_complex=True,
property_to_vars=True,
guess_unit=True,
normalize=True,
make_matrix=True,
merge=True,
verbose=True)
d = DataBlock()
d.comments = Comments([])
fi = DimSweep("freq", [0e9, 1e9, 2e9])
d.S = hfarray([[[1 + 1j, 1 + 2j], [2 + 1j, 2 + 2j]]] * 3,
dims=(
fi,
DimMatrix_i("i", 2),
DimMatrix_j("j", 2),
))
save_touchstone(d, "tests/testdata/touchstone/savetest/res_1.txt")
d1 = DataBlock()
d1.b = hfarray([2], dims=(ia, ))
d2 = DataBlock()
d2.b = hfarray([3], dims=(ia, ))
fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5"
with hdf5context(fname, mode="w") as fil:
save_hdf5_v01(d1, fil)
self.assertRaises(Exception, append_hdf5, d2, fil)
# append_hdf5(d2, fil)
fname.unlink()
if __name__ == '__main__':
d = DataBlock()
d.comments = Comments(["Hej=10", "Svejs=11"])
dims = (
DimSweep("f", 3, outputformat="%.1f"),
DimSweep("i", 2, outputformat="%.0f"),
DimSweep("j", 2, outputformat="%.0f"),
)
d.c = hfarray(
[[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]],
dims=dims,
outputformat="%.2f")
fname = testpath / "testdata/sp-data/savetest/res_3.txt"
hftools.file_formats.spdata.save_spdata(d, fname)
fname2 = testpath / "testdata/sp-data/a.txt"
d2 = hftools.file_formats.read_spdata(fname2, verbose=False)
