def merge_blocks(blocks, hyper=False, indexed=False): db = DataBlock() dimpartialgroups = OrderedDict() for b in blocks: parts = get_partials(b) partgroup = dimpartialgroups.setdefault(tuple(parts.keys()), {}) for k, v in parts.items(): partgroup.setdefault(k, []).extend(v) for idx, dims in enumerate(dimpartialgroups.values(), 1): for k, v in dims.items(): dims = (DimSweep("INDEX%s" % idx, len(v)),) db[k] = hfarray([x.data[0] for x in v], dims=dims, unit=v[0].unit) varnames = set() for b in blocks: for k, v in b.ivardata.items(): if k not in db: db[k] = v for k in b.vardata.keys(): varnames.add(k) for vname in varnames: v = [] for b in blocks: if vname not in b: continue partials = get_partials(b) v.append(b[vname]) if v: k = tuple(partials.keys()) if k: ri = (db[tuple(dimpartialgroups[k].keys())[0]].dims[0],) else: ri = tuple() value = hfarray(v, dims=ri + v[0].dims, unit=v[0].unit) if v[0].dims and isinstance(v[0].dims[0], DimSweep): value = value.reorder_dimensions(v[0].dims[0]) db[vname] = value cmt = Comments() for block in blocks: if block.comments: cmt.extend(block.comments) db.comments = cmt db.blockname = blocks[0].blockname if hyper: for vnames in dimpartialgroups.keys(): if vnames: hyperindex = db[vnames[0]].dims[0] db = db.hyper(vnames, hyperindex, all=True) db = db.squeeze() return db
def merge_blocks(blocks, hyper=False, indexed=False): db = DataBlock() dimpartialgroups = OrderedDict() for b in blocks: parts = get_partials(b) partgroup = dimpartialgroups.setdefault(tuple(parts.keys()), {}) for k, v in parts.items(): partgroup.setdefault(k, []).extend(v) for idx, dims in enumerate(dimpartialgroups.values(), 1): for k, v in dims.items(): dims = (DimSweep("INDEX%s" % idx, len(v)), ) db[k] = hfarray([x.data[0] for x in v], dims=dims, unit=v[0].unit) varnames = set() for b in blocks: for k, v in b.ivardata.items(): if k not in db: db[k] = v for k in b.vardata.keys(): varnames.add(k) for vname in varnames: v = [] for b in blocks: if vname not in b: continue partials = get_partials(b) v.append(b[vname]) if v: k = tuple(partials.keys()) if k: ri = (db[tuple(dimpartialgroups[k].keys())[0]].dims[0], ) else: ri = tuple() value = hfarray(v, dims=ri + v[0].dims, unit=v[0].unit) if v[0].dims and isinstance(v[0].dims[0], DimSweep): value = value.reorder_dimensions(v[0].dims[0]) db[vname] = value cmt = Comments() for block in blocks: if block.comments: cmt.extend(block.comments) db.comments = cmt db.blockname = blocks[0].blockname if hyper: for vnames in dimpartialgroups.keys(): if vnames: hyperindex = db[vnames[0]].dims[0] db = db.hyper(vnames, hyperindex, all=True) db = db.squeeze() return db
def read_hdf5(h5file, name="datablock", **kw): if isinstance(h5file, string_types): fil = h5py.File(h5file, "r") else: fil = h5file db = DataBlock() grp = fil[name] blockname = grp.attrs["Blockname"] if blockname.lower() == "none": blockname = None db.blockname = blockname comments = grp["Comments"] if "fullcomments" in comments and len(comments["fullcomments"]): db.comments = Comments([ cast_unicode(x).strip() for x in np.array(comments["fullcomments"]) ]) else: db.comments = Comments() ivardata = grp["ivardata"] vardata = grp["vardata"] for k in ivardata: v = ivardata[k] datadtype = v.attrs[r"info\dtype"] or None dimcls = dims_dict.get(v.attrs[r"info\class"], DimRep) unit = str(v.attrs.get(r"info\unit", "none")) if unit.lower() == "none": unit = None vdata = np.array(np.array(v), dtype=datadtype) dim = dimcls(k, vdata, unit=unit) db[k] = dim for k in vardata: v = vardata[k] datadtype = v.attrs[r"data\dtype"] or None dims = tuple(db.ivardata[cast_unicode(dimname)] for dimname in v.attrs[r"info\name"]) unit = cast_unicode(v.attrs.get(r"data\unit", "none")) if unit.lower() == "none": unit = None db[k] = hfarray(np.array(v), dtype=datadtype, dims=dims, unit=unit) if isinstance(h5file, string_types): fil.close() if kw.get("property_to_vars", False): db.values_from_property() return db
def test_4(self): d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dim = DimSweep("f", 3, outputformat="%.1f") d.freq = dim fname = testpath / "testdata/hdf5/v02/savetest/res_4.hdf5" self.savefun[0](d, fname) fname.unlink()
def test_2(self): d = DataBlock() d.comments = Comments(["Hej=10"]) # import pdb;pdb.set_trace() d.b = hfarray([2], dims=(DimSweep("a", 1), )) fname = testpath / "testdata/hdf5/v02/savetest/res_2.hdf5" self.savefun[0](d, fname) fname.unlink()
def test_9(self): d = DataBlock() d.comments = Comments(["Hej=10"]) d.b = hfarray([2], dims=(DimSweep("a", 1), ), unit="V") fname = testpath / "testdata/hdf5/v01/savetest/res_9.hdf5" self.savefun[0](d, fname) d2 = readfun(fname) self.assertEqual(d2.b.unit, "V") fname.unlink()
def merge_blocks_do_hyper(blocks): #import pdb;pdb.set_trace() outdata = DataBlock() data = merge_blocks_to_association_list(blocks) ivars = {} for b in blocks: for vname, v in b.ivardata.items(): if vname not in ivars: ivars[vname] = v free_vars = set() for vname, assoc in data.items(): free_vars.add(zip(*assoc)[0]) free_vars = list(free_vars) for vname, assoc in data.items(): v, indexvars, dim = merge_variable(assoc) outdata[vname] = v for iname, value in indexvars.items(): outdata.ivardata[iname] = value.dims[0] for v in outdata.vardata.keys(): if v in outdata.ivardata: del outdata.vardata[v] for v in outdata.vardata.values(): for k in v.dims: if k.name in outdata.ivardata: if ((isinstance(k, DimRep) and not isinstance(outdata.ivardata[k.name], DimRep))): outdata.replace_dim(outdata.ivardata[k.name], k) cmt = Comments() for block in blocks: if block.comments: cmt.extend(block.comments) outdata.comments = cmt outdata.blockname = blocks[0].blockname for vname, v in ivars.items(): if vname not in outdata: outdata[vname] = v #import pdb;pdb.set_trace() return outdata
def test_5(self): d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dim = DimSweep("f", 3, outputformat="%.1f") d.freq = dim d.date = hfarray("2012-08-13 08:03:01", dtype="datetime64[us]") fname = testpath / "testdata/hdf5/v02/savetest/res_5.hdf5" self.savefun[0](d, fname) d2 = readfun(fname) fname.unlink() self.assertEqual(d2.date, d.date)
def test_save_2(self): d = hftools.file_formats.read_citi(testpath / "testdata/citi/test1.citi") d.comments = Comments(["klasjd:1"]) d.comments.fullcomments = ["kljlk"] fname = testpath / "testdata/citi/savetest/test1.citi" hftools.file_formats.save_citi(d, fname) e = hftools.file_formats.read_citi(fname) self.assertAllclose(d.S, e.S) fname.unlink()
def test_10(self): d = DataBlock() d.comments = Comments(["Hej=10"]) # import pdb;pdb.set_trace() d.b = hfarray([2], dims=(DimSweep("a", 1), ), unit="V") d.b.outputformat = "" fname = testpath / "testdata/hdf5/v02/savetest/res_10.hdf5" self.savefun[0](d, fname) d2 = readfun(fname) self.assertEqual(d2.b.outputformat, "%d") fname.unlink()
def test_11(self): ia = DimSweep("a", 1) d = DataBlock() d.comments = Comments(["Hej=10"]) # import pdb;pdb.set_trace() d.b = hfarray([u"kalle"], dims=(ia, ), unit="V") d.c = hfarray(u"kalle") fname = testpath / "testdata/hdf5/v02/savetest/res_11.hdf5" self.savefun[0](d, fname) d2 = readfun(fname) self.assertEqual(d2.b[0], u"kalle") self.assertEqual(d2.c, u"kalle") fname.unlink()
def test_2(self): d = DataBlock() d.comments = Comments(["Hej=10"]) # import pdb;pdb.set_trace() d.b = hfarray([2], dims=(DimSweep("a", 1), )) f1 = testpath / "testdata/sp-data/savetest/res_2.txt" f2 = testpath / "testdata/sp-data/savetest/facit_2.txt" hftools.file_formats.spdata.save_spdata(d, f1) with open(f1) as resfil: with open(f2) as facitfil: for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)): msg = "\nFailed on line %d\n result: %r\n facit: %r" args = (idx + 1, rad1, rad2) self.assertEqual(rad1, rad2, msg=msg % args)
def test_4(self): d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dim = DimSweep("f", 3, outputformat="%.1f") d.freq = dim fname = testpath / "testdata/sp-data/savetest/res_4.txt" facitfname = testpath / "testdata/sp-data/savetest/facit_4.txt" hftools.file_formats.spdata.save_spdata(d, fname) with open(fname) as resfil: with open(facitfname) as facitfil: for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)): msg = "\nFailed on line %d\n result: %r\n facit: %r" args = (idx + 1, rad1, rad2) self.assertEqual(rad1, rad2, msg=msg % args)
def parse_blocks(self, stream): # raise Exception("HANTERAR INTE KOMMENTARER INNE I MANYOPTIONAL") comments = [] comments.extend(ManyOptional("COMMENT")(stream)) citifileversion = Optional("CITIFILE")(stream) comments.extend(ManyOptional("COMMENT")(stream)) name = One("NAME", "Must have END after DATA")(stream) comments.extend(ManyOptional("COMMENT")(stream)) varnames = ManyOptional("VARDEF")(stream) comments.extend(ManyOptional("COMMENT")(stream)) datanames = ManyOptional("DATADEF")(stream) comments.extend(ManyOptional("COMMENT")(stream)) block = DataBlock() block.blockname = name[0] for name, typ, N in varnames: N = int(N) errmsg = "Missing VAR_LIST_BEGIN or SEG_LIST_BEGIN" tagname, = OneOf(["VAR_LIST_BEGIN", "SEG_LIST_BEGIN"], errmsg)(stream) if tagname.startswith("VAR_LIST"): datalist = handle_data(ManyOptional("DATA")(stream), typ) block[name] = DimSweep(name, datalist) One("VAR_LIST_END", "Missing VAR_LIST_END")(stream) elif tagname.startswith("SEG"): # pragma: no branch datalist, = One("SEG", "Missing SEG")(stream) _, start, stop, step = datalist.strip().split() block[name] = DimSweep(name, np.linspace(float(start), float(stop), int(step))) One("SEG_LIST_END", "Missing SEG_LIST_END")(stream) else: # pragma: no cover pass comments.extend(ManyOptional("COMMENT")(stream)) for idx, com in enumerate(comments): if com.startswith("CONSTANT TIME"): date = tuple(com.strip().split()[2:]) comments[idx] = "Measurement time: %s-%s-%s %s:%s:%s" % date block.comments = Comments(comments) shape = tuple(block.ivardata[i[0]].data.shape[0] for i in varnames) dims = tuple(block.ivardata[i[0]] for i in varnames) for name, typ in datanames: One("DATA_LIST_BEGIN", "Missing BEGIN")(stream) datalist = array(handle_data(ManyOptional("DATA")(stream), typ)) datalist.shape = shape block[name] = hfarray(datalist, dims=dims) One("DATA_LIST_END", "Missing END")(stream) yield block
def test_3(self): d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dims = ( DimSweep("f", 3, outputformat="%.1f"), DimSweep("i", 2, outputformat="%.0f"), DimSweep("j", 2, outputformat="%.0f"), ) d.c = hfarray( [[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]], dims=dims, outputformat="%.2f") fname = testpath / "testdata/hdf5/v02/savetest/res_3.hdf5" self.savefun[0](d, fname) fname.unlink()
def parse_blocks(self, stream): for comments, header, data in stream: db = DataBlock() db.comments = Comments(comments) header = header[0].strip().split("\t") Nhead = len(header) #data = np.array(data) if Nhead != len(data[0]): msg = "Different number of header variables "\ "from data columns" raise SPDataIOError(msg) output = DataDict() for varname, column in zip(header, zip(*data)): output.setdefault(varname, []).append(column) for varname in output: data = output[varname] if len(data) > 1: output[varname] = np.array(output[varname], order="F").T else: output[varname] = np.array(output[varname][0]) freq = DimSweep(header[0], output[header[0]]) db[header[0]] = freq for x in output.keys()[1:]: if output[x].ndim == 1: db[x] = hfarray(output[x], dims=(freq, )) else: repdim = DimRep("rep", output[x].shape[1]) db[x] = hfarray(output[x], dims=(freq, repdim)).squeeze() remove = [] for vname in db.comments.property: if vname[:1] == "@": unit = db.comments.property[vname].unit data = [float(db.comments.property[vname])] db[vname[1:]] = DimPartial(vname[1:], data=data, unit=unit) remove.append(vname) for v in remove: del db.comments.property[v] db.comments.fullcomments = [ com for com in db.comments.fullcomments if not com.startswith("@") ] yield db
def parse_blocks(self, stream): for comments, vars, blockname, header, attribs, data in stream: db = DataBlock() db.blockname = blockname = blockname[0] #self.header = header = header[0] db.comments = Comments(comments) #data = numpy.array(data) dd = self.proc_data(header, data[:]) for vname in dd: db[vname] = dd[vname] for var in vars: vi = self.proc_var(var) db[vi.name] = vi for var in attribs: name, value = self.proc_attrib(var) db[name] = value self.make_matrices(db, sum(header, [])) yield db
def test_3(self): d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dims = ( DimSweep("f", 3, outputformat="%.1f"), DimSweep("i", 2, outputformat="%.0f"), DimSweep("j", 2, outputformat="%.0f"), ) d.c = hfarray( [[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]], dims=dims, outputformat="%.2f") f1 = testpath / "testdata/sp-data/savetest/res_3.txt" f2 = testpath / "testdata/sp-data/savetest/facit_3.txt" hftools.file_formats.spdata.save_spdata(d, f1) with open(f1) as resfil: with open(f2) as facitfil: for idx, (rad1, rad2) in enumerate(zip(resfil, facitfil)): msg = "\nFailed on line %d\n result: %r\n facit: %r" args = (idx + 1, rad1, rad2) self.assertEqual(rad1, rad2, msg=msg % args)
def simple_merge_blocks(blocks): if len(blocks) == 1: return blocks[0] ri = DimRep("INDEX", len(blocks)) out = DataBlock() #import pdb;pdb.set_trace() for k in blocks[0].vardata: out[k] = hfarray([b[k] for b in blocks], dims=(ri, ) + blocks[0][k].dims) if blocks[0][k].dims: out[k] = out[k].reorder_dimensions(blocks[0][k].dims[0]) for k in blocks[0].ivardata: if isinstance(blocks[0].ivardata[k], DimPartial): out[k] = hfarray([b.ivardata[k].data[0] for b in blocks], dims=(ri, )) c = [] for b in blocks: if b.comments is not None: c.extend(b.comments.fullcomments) out.comments = Comments() out.comments.fullcomments = c return out
def parse_blocks(self, stream): comments = [] info = None datalist = [] for token, lineno, rad in stream: if token == "Comments": comments.append(rad) elif token == "Info": if info is None: info = rad else: msg = "Second # info at lineno: %d" % lineno raise TouchstoneError(msg) else: datalist.append(list(map(float, rad.split()))) if info is None: raise TouchstoneError("No # info line in file") comments = Comments(comments) f, data, fn, noisedata = proc_data(datalist) out = proc_info(info, f, data, fn, noisedata) #HZ S RI R 50 #comments.add_from_comment("!INFO:#%s"%(" ".join(info))) out.comments = comments yield out
make_complex=make_complex, property_to_vars=property_to_vars, guess_unit=guess_unit, normalize=normalize, make_matrix=make_matrix, merge=merge, verbose=verbose) return res if __name__ == "__main__": a = read_touchstone("tests/testdata/touchstone/test4.s2p", make_complex=True, property_to_vars=True, guess_unit=True, normalize=True, make_matrix=True, merge=True, verbose=True) d = DataBlock() d.comments = Comments([]) fi = DimSweep("freq", [0e9, 1e9, 2e9]) d.S = hfarray([[[1 + 1j, 1 + 2j], [2 + 1j, 2 + 2j]]] * 3, dims=( fi, DimMatrix_i("i", 2), DimMatrix_j("j", 2), )) save_touchstone(d, "tests/testdata/touchstone/savetest/res_1.txt")
d1 = DataBlock() d1.b = hfarray([2], dims=(ia, )) d2 = DataBlock() d2.b = hfarray([3], dims=(ia, )) fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5" with hdf5context(fname, mode="w") as fil: save_hdf5_v01(d1, fil) self.assertRaises(Exception, append_hdf5, d2, fil) # append_hdf5(d2, fil) fname.unlink() if __name__ == '__main__': d = DataBlock() d.comments = Comments(["Hej=10", "Svejs=11"]) dims = ( DimSweep("f", 3, outputformat="%.1f"), DimSweep("i", 2, outputformat="%.0f"), DimSweep("j", 2, outputformat="%.0f"), ) d.c = hfarray( [[[1, 2], [3, 4]], [[10, 20], [30, 40]], [[10, 20], [30, 40]]], dims=dims, outputformat="%.2f") fname = testpath / "testdata/sp-data/savetest/res_3.txt" hftools.file_formats.spdata.save_spdata(d, fname) fname2 = testpath / "testdata/sp-data/a.txt" d2 = hftools.file_formats.read_spdata(fname2, verbose=False)