def setUp(self): self.db = db = DataBlock() Sdims = (DimRep("freq", [1, 2]), DimRep("r", [1]), DimMatrix_i("i", 2), DimMatrix_j("j", 2)) db.S = hfarray(np.array([[11, 12], [21, 22]])[None, None, :, :] * np.array([[10], [20]])[..., None, None], dims=Sdims) db.V = hfarray([1.23], Sdims[1:2]) db.Y = hfarray([1.23], (DimRep("k", [1]), ))
def setUp(self): d = DataBlock() fi = DimSweep("Freq[Hz]", 3) gi = DimRep("g", 2) hi = DimRep("h", 3) ji = DimRep("j", 4) dims = (fi, ) d["a"] = hfarray([3, 2, 1], dims=dims, outputformat="%.5f", unit="V") d["b"] = hfarray([30, 10], dims=(gi, ), outputformat="%.7f", unit="A") d["c"] = hfarray([[30, 20, 10, 40], [20, 30, 40, 10]], dims=(gi, hi), outputformat="%.7f", unit="A") self.d = d
def test_1(self): db = DataBlock() Sdims = (DimRep("freq", [1, 2]), DimRep("r", [1]), DimMatrix_i("i", 2), DimMatrix_j("j", 2)) db.S = hfarray( np.array([[11, 12], [21, 22]])[np.newaxis, np.newaxis, :, :] * np.array([[10], [20]])[..., np.newaxis, np.newaxis], dims=Sdims) db.V = hfarray([1.23], Sdims[1:2]) db.Y = hfarray([1.23], (DimRep("k", [1]), )) out = dset.subset_datablock_by_dims( dset.convert_matrices_to_elements(db), Sdims[:-2]) self.assertTrue("V" in out) self.assertTrue("S11" in out) self.assertTrue("S12" in out) self.assertTrue("S21" in out) self.assertTrue("S22" in out) self.assertFalse("Y" in out)
def test_cant_append(self): i1 = DimSweep("a", 1, unit="s", outputformat="") i2 = DimSweep("INDEX", 2, unit="s") iexpand = DimRep("INDEX", 1, unit="s", outputformat="") d1 = DataBlock() d1.b = hfarray([2], dims=(i1, ), outputformat="") d1.c = hfarray(3) d2 = DataBlock() d2.b = hfarray([3], dims=(i1, )) fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5" with hdf5context(fname, mode="w") as fil: savefun(d1, fil, expandable=True, expanddim=iexpand) self.assertRaises(ValueError, append_hdf5, d2, fil) fname.unlink()
def parse_blocks(self, stream): for comments, header, data in stream: db = DataBlock() db.comments = Comments(comments) header = header[0].strip().split("\t") Nhead = len(header) #data = np.array(data) if Nhead != len(data[0]): msg = "Different number of header variables "\ "from data columns" raise SPDataIOError(msg) output = DataDict() for varname, column in zip(header, zip(*data)): output.setdefault(varname, []).append(column) for varname in output: data = output[varname] if len(data) > 1: output[varname] = np.array(output[varname], order="F").T else: output[varname] = np.array(output[varname][0]) freq = DimSweep(header[0], output[header[0]]) db[header[0]] = freq for x in output.keys()[1:]: if output[x].ndim == 1: db[x] = hfarray(output[x], dims=(freq, )) else: repdim = DimRep("rep", output[x].shape[1]) db[x] = hfarray(output[x], dims=(freq, repdim)).squeeze() remove = [] for vname in db.comments.property: if vname[:1] == "@": unit = db.comments.property[vname].unit data = [float(db.comments.property[vname])] db[vname[1:]] = DimPartial(vname[1:], data=data, unit=unit) remove.append(vname) for v in remove: del db.comments.property[v] db.comments.fullcomments = [ com for com in db.comments.fullcomments if not com.startswith("@") ] yield db
def merge_variable(association_list): names, indices, association_list =\ association_list_in_hypercube_order(association_list) innershape = tuple(len(x) for x in indices) innerdims = tuple(DimRep(name, i) for name, i in zip(names, indices)) if len(association_list) == 1: #pdb.set_trace() return association_list[0][1], {}, None basedims = association_list[0][1].dims prototype = association_list[0][1] if prototype.dtype.type in (np.unicode_, np.str_): length = max(int(x[1].dtype.str[2:]) for x in association_list) proto_dtype = np.dtype((prototype.dtype.type, length)) else: proto_dtype = prototype.dtype newshape = prototype.shape[:1] + innershape + prototype.shape[1:] result = empty(newshape, dtype=proto_dtype) unit = [] for idx, x in association_list: unit.append(getattr(x, "unit", None)) if prototype.ndim: result[(slice(None, None, None), ) + idx] = x else: if x.ndim == 0: result[idx] = x.flatten()[0] else: result[idx] = x if len(set(unit)) == 1: unit = unit.pop() else: unit = None v = hfarray(result, dims=basedims[:1] + innerdims + basedims[1:], unit=unit) if len(association_list[0][0]) == 0: indexvars = {} else: indexvars = dict((x.name, hfarray(x)) for x in innerdims) return v, indexvars, innerdims
def save_hdf5_handle(db, filehandle, expandable=False, expanddim=None, **kw): if expanddim is None: expanddim = DimRep("INDEX", 1) filehandle.attrs["hftools file version"] = "0.2" for k, v in db.ivardata.items(): ek = escape_varname(k) create_dataset(filehandle, ek, v.data) filehandle[ek].attrs["dimtype"] = v.__class__.__name__ if v.unit: filehandle[ek].attrs["unit"] = v.unit if v.outputformat: filehandle[ek].attrs["outputformat"] = v.outputformat if expandable: data = np.array(expanddim.data) filehandle.create_dataset(expanddim.name, data=data, maxshape=(None, )) filehandle[expanddim.name].attrs["dtype"] = data.dtype.str expanddim_class = expanddim.__class__.__name__ filehandle[expanddim.name].attrs["dimtype"] = expanddim_class if expanddim.unit: filehandle[expanddim.name].attrs["unit"] = expanddim.unit if expanddim.outputformat: outputformat = expanddim.outputformat filehandle[expanddim.name].attrs["outputformat"] = outputformat for k, v in db.vardata.items(): ek = escape_varname(k) if expandable and v.ndim == 0: v = v.add_dim(expanddim) elif expandable: v = v.add_dim(expanddim, 1) else: pass create_dataset(filehandle, ek, v, expandable=expandable) for idx, dv in enumerate(v.dims): filehandle[ek].dims.create_scale(filehandle[dv.name]) filehandle[ek].dims[idx].attach_scale(filehandle[dv.name]) filehandle[ek].attrs["arraytype"] = v.__class__.__name__ if v.unit: filehandle[ek].attrs["unit"] = v.unit if v.outputformat: filehandle[ek].attrs["outputformat"] = v.outputformat
def setUp(self): d = DataBlock() gi = DimSweep("Index", 6) ri = DimRep("Rep", 2) dims = (gi, ) d["a"] = hfarray([1, 1, 2, 2, 3, 3], dims=dims, outputformat="%.5f", unit="V") d["b"] = hfarray([10, 20, 10, 20, 10, 20], dims=(gi, ), outputformat="%.7f", unit="A") d["c"] = hfarray([10.1, 20.1, 10.2, 20.2, 10.3, 20.3], dims=(gi, ), outputformat="%.7f", unit="A") d["d"] = hfarray([17, 19], dims=(ri, ), outputformat="%.7f", unit="A") self.d = d
def append_hdf5(db, filehandle, expanddim=None, **kw): key = "hftools file version" if key not in filehandle.attrs or filehandle.attrs[key] != "0.2": raise Exception("Can only append to hftools version 0.2") if expanddim is None: expanddim = DimRep("INDEX", 1) idx = filehandle[expanddim.name][-1] + 1 expand_dataset(filehandle[expanddim.name], 1, 0) filehandle[expanddim.name].value[-1] = idx for k, v in filehandle.items(): if k in db.ivardata or k == expanddim.name: continue if k not in db.vardata: raise ValueError("Variable %r missing in db, can not append" % k) diskdata = filehandle[k] ax = diskdata.maxshape.index(None) expand_dataset(diskdata, 1, ax) s = (slice(None),) * ax + (diskdata.shape[ax] - 1,) diskdata[s] = db[k]
def simple_merge_blocks(blocks): if len(blocks) == 1: return blocks[0] ri = DimRep("INDEX", len(blocks)) out = DataBlock() #import pdb;pdb.set_trace() for k in blocks[0].vardata: out[k] = hfarray([b[k] for b in blocks], dims=(ri, ) + blocks[0][k].dims) if blocks[0][k].dims: out[k] = out[k].reorder_dimensions(blocks[0][k].dims[0]) for k in blocks[0].ivardata: if isinstance(blocks[0].ivardata[k], DimPartial): out[k] = hfarray([b.ivardata[k].data[0] for b in blocks], dims=(ri, )) c = [] for b in blocks: if b.comments is not None: c.extend(b.comments.fullcomments) out.comments = Comments() out.comments.fullcomments = c return out
def test_2(self): i1 = DimSweep("a", 1, unit="s", outputformat="") i2 = DimSweep("INDEX", 2, unit="s") iexpand = DimRep("INDEX", 1, unit="s", outputformat="") d1 = DataBlock() d1.b = hfarray([2], dims=(i1, ), outputformat="") d1.c = hfarray(3) d2 = DataBlock() d2.b = hfarray([3], dims=(i1, )) d2.c = hfarray(2) fname = testpath / "testdata/hdf5/v02/savetest/res_1.hdf5" with hdf5context(fname, mode="w") as fil: savefun(d1, fil, expandable=True, expanddim=iexpand) append_hdf5(d2, fil, expanddim=iexpand) d = readfun(fname) self.assertAllclose(hfarray([[2, 3]], dims=(i1, i2)), d.b) self.assertAllclose(hfarray([3, 2], dims=(i2, )), d.c) self.assertEqual(d.ivardata["a"].unit, "s") self.assertEqual(d.ivardata["INDEX"].unit, "s") self.assertEqual(d.ivardata["a"].outputformat, "%d") self.assertEqual(d.ivardata["INDEX"].outputformat, "%d") self.assertEqual(d.b.outputformat, "%d") fname.unlink()
def test_2(self): d = DataBlock() d.FILENAME = hfarray(["foo.txt", "bar.txt"], dims=(DimRep("FILEINDEX", [0, 1]), )) d.blockname = "baz.txt" self.assertEqual(d.blockname, "baz.txt")