Example #1
0
    def setUp(self):
        self.temp = TempFile()

        ad.init_noxml()

        ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
        g = ad.declare_group("temperature", "", ad.FLAG.YES)
        ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
                      "size,NX", "0,0")
        self.msg = "this is a test"
        self.unit = "C"
        ## attr must be <varpath> + / + something without / (line 857, common_read.c)
        ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string,
                            self.msg, "")
        ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string,
                            self.unit, "")
        ad.select_method(g, "POSIX1", "verbose=3", "")

        fd = ad.open("temperature", self.temp.path, "w")
        self.NX = 10
        self.size = 2
        groupsize = 4 + 4 + 8 * self.size * self.NX
        t = np.array(range(self.NX * self.size), dtype=np.float64)
        self.tt = t.reshape((self.size, self.NX))
        ad.set_group_size(fd, groupsize)
        ad.write_int(fd, "NX", self.NX)
        ad.write_int(fd, "size", self.size)
        ad.write(fd, "temperature", self.tt)
        ad.close(fd)

        ad.finalize()

        self.f = ad.file(self.temp.path)
Example #2
0
    def setUp(self):
        self.temp = TempFile()

        ad.init_noxml()

        g = ad.declare_group("temperature", "", ad.FLAG.YES)
        ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
                      "size,NX", "0,0")
        self.msg = "this is a test"
        ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
        ad.select_method(g, "POSIX1", "verbose=3", "")

        for i in range(5):
            mode = "a"
            if i == 0: mode = "w"
            fd = ad.open("temperature", self.temp.path, mode)
            self.NX = 10
            self.size = 2
            ##groupsize =  4 + 4 + 8 * self.size * self.NX
            t = np.array(list(range(self.NX * self.size)),
                         dtype=np.float64) + 100 * i
            self.tt = t.reshape((self.size, self.NX))
            ##ad.set_group_size(fd, groupsize)
            ad.write_int(fd, "NX", self.NX)
            ad.write_int(fd, "size", self.size)
            ad.write(fd, "temperature", self.tt)
            ad.close(fd)

        ad.finalize()

        self.f = ad.file(self.temp.path)
    def setUp(self):
        self.temp = TempFile()

        ad.init_noxml()

        ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
        g = ad.declare_group("temperature", "", ad.FLAG.YES)
        ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
        self.msg = "this is a test"
        ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
        ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, "C", "")
        ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string, "description", "")
        ad.define_attribute(g, "/subgroup/subsubgroup/otherattr", "", ad.DATATYPE.string, "another", "")
        ad.define_var(g, "/subgroup/subsubgroup/othervar", "", ad.DATATYPE.integer, "", "", "")
        ad.select_method(g, "POSIX1", "verbose=3", "")

        fd = ad.open("temperature", self.temp.path, "w")
        self.NX = 10
        self.size = 2
        groupsize =  4 + 4 + 8 * self.size * self.NX + 4
        t = np.array(range(self.NX * self.size), dtype=np.float64)
        self.tt = t.reshape((self.size, self.NX))
        ad.set_group_size(fd, groupsize)
        ad.write_int(fd, "NX", self.NX)
        ad.write_int(fd, "size", self.size)
        ad.write(fd, "temperature", self.tt)
        ad.write_int(fd, "/subgroup/subsubgroup/othervar", 99)
        ad.close(fd)

        ad.finalize()

        self.f = ad.file(self.temp.path)
Example #4
0
    def setUp(self):
        self.temp = TempFile()

        ad.init_noxml()

        ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
        g = ad.declare_group("temperature", "", ad.FLAG.YES)
        ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
        self.msg = "this is a test"
        self.unit = "C"
        ## attr must be <varpath> + / + something without / (line 857, common_read.c)
        ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string, self.msg, "")
        ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, self.unit, "")
        ad.select_method(g, "POSIX1", "verbose=3", "")

        fd = ad.open("temperature", self.temp.path, "w")
        self.NX = 10
        self.size = 2
        groupsize =  4 + 4 + 8 * self.size * self.NX
        t = np.array(range(self.NX * self.size), dtype=np.float64)
        self.tt = t.reshape((self.size, self.NX))
        ad.set_group_size(fd, groupsize)
        ad.write_int(fd, "NX", self.NX)
        ad.write_int(fd, "size", self.size)
        ad.write(fd, "temperature", self.tt)
        ad.close(fd)

        ad.finalize()

        self.f = ad.file(self.temp.path)
Example #5
0
    def setUp(self):
        self.temp = TempFile()

        ad.init_noxml()

        ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
        g = ad.declare_group("temperature", "", ad.FLAG.YES)
        ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
                      "size,NX", "0,0")
        ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
                      "size,NX", "0,0")
        self.msg = "this is a test"
        ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
        ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, "C",
                            "")
        ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string,
                            "description", "")
        ad.define_attribute(g, "/subgroup/subsubgroup/otherattr", "",
                            ad.DATATYPE.string, "another", "")
        ad.define_var(g, "/subgroup/subsubgroup/othervar", "",
                      ad.DATATYPE.integer, "", "", "")
        ad.select_method(g, "POSIX1", "verbose=3", "")

        fd = ad.open("temperature", self.temp.path, "w")
        self.NX = 10
        self.size = 2
        groupsize = 4 + 4 + 8 * self.size * self.NX + 4
        t = np.array(range(self.NX * self.size), dtype=np.float64)
        self.tt = t.reshape((self.size, self.NX))
        ad.set_group_size(fd, groupsize)
        ad.write_int(fd, "NX", self.NX)
        ad.write_int(fd, "size", self.size)
        ad.write(fd, "temperature", self.tt)
        ad.write_int(fd, "/subgroup/subsubgroup/othervar", 99)
        ad.close(fd)

        ad.finalize()

        self.f = ad.file(self.temp.path)
    def write_scalar(self, adtype, val, varname='val'):
        ad.init_noxml()
        g = ad.declare_group("group", "", ad.FLAG.YES)
        ad.define_var(g, varname, "", adtype, "", "", "")
        ad.select_method(g, "POSIX1", "", "")

        if adtype == ad.DATATYPE.string:
            dtype = ad.adios2npdtype(adtype, len(str(val)))
            npval = np.array(val, dtype=dtype)
        else:
            dtype = ad.adios2npdtype(adtype)
            npval = np.array(val, dtype=dtype)

        fd = ad.open("group", self.temp.path, "w")
        ##ad.set_group_size(fd, npval.nbytes)
        ad.write(fd, varname, val, dtype)
        ad.close(fd)
        ad.finalize()

        f = ad.file(self.temp.path)
        v = f.vars['val']
        self.assertEqual(v.read(), npval)
Example #7
0
    def write(self, msg):

        doid = msg['doid']
        column = msg['column']
        rows = msg['rows']
        shape = msg['shape']
        rowid = msg['row']
        datatype = msg['datatype']
        data = msg['data']
        filename = self.filepath + '/' + doid

        dimension = [rows] + copy.copy(shape)
        global_dimension = copy.copy(dimension)
        global_dimension[0] = self.maxrows
        offset = [0] * len(dimension)
        offset[0] = rowid

        ad.init_noxml()
        ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, self.buffersize)
        adios_group = ad.declare_group("shore", "", ad.FLAG.YES)
        local_dimension = str(dimension)[1:-1]
        global_dimension = str(global_dimension)[1:-1]
        offset = str(offset)[1:-1]

        ad.select_method(adios_group, "POSIX", "", "")
        ad.define_var(adios_group, column, "",
                      self.dtype_numpy2adios[datatype], local_dimension,
                      global_dimension, offset)
        adios_file = ad.open("shore", filename, 'u')
        group_size = 1000000
        if data.nbytes > group_size:
            group_size = data.nbytes
        ad.set_group_size(adios_file, group_size)
        ad.write(adios_file, column, data, datatype)
        ad.close(adios_file)
        ad.finalize()

        return True
    def write_scalar(self, adtype, val, varname='val'):
        ad.init_noxml()
        ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
        g = ad.declare_group("group", "", ad.FLAG.YES)
        ad.define_var(g, varname, "", adtype, "", "", "")
        ad.select_method(g, "POSIX1", "", "")

        if adtype == ad.DATATYPE.string:
            dtype = ad.adios2npdtype(adtype, len(str(val)))
            npval = np.array(val, dtype=dtype)
        else:
            dtype = ad.adios2npdtype(adtype)
            npval = np.array(val, dtype=dtype)

        fd = ad.open("group", self.temp.path, "w")
        ad.set_group_size(fd, npval.nbytes)
        ad.write(fd, varname, val, dtype)
        ad.close(fd)
        ad.finalize()

        f = ad.file(self.temp.path)
        v = f.vars['val']
        self.assertEqual(v.read(), npval)
Example #9
0
def getTimeAv(dataDir, simName, varName, iFrame, fFrame, p, b, saveAv, tAvDir):
    #.Check or create post data directory.
    checkMkdir(tAvDir)
    #.Check if time average file already exists.
    tAvFile = tAvDir + simName + '_' + varName + '_TimeAv' + str(
        iFrame) + '-' + str(fFrame) + '.bp'
    if not os.path.isfile(tAvFile):
        #.Compute time average and store it in new file.
        fileName = dataDir + simName + '_' + varName + '_%d.bp'
        x, gridDim, nx, lx, dx = getGrid(fileName % iFrame,
                                         p,
                                         b,
                                         location='center')

        q0AvT = np.zeros(nx)
        for nFr in range(iFrame, fFrame + 1):
            #.Read 3D data into q0.
            q0AvT = np.add(q0AvT,
                           np.squeeze(getInterpData(fileName % nFr, p, b)))

        q0AvT = np.divide(q0AvT, float(fFrame - iFrame + 1))

        if saveAv:
            #.Save time average to a file for reuse.
            print(" ")
            print("Saving time average in " + tAvFile + " ...")
            #.Function to write DG coefficients to Gkeyll-style ADIOS file.
            sNumCells = ""
            sOffsets = ""
            for i in range(np.size(nx)):
                sNumCells += "{:d},".format(int(nx[i]))
                sOffsets += "0,"
            #.ADIOS init.
            ad.init_noxml()
            ad.set_max_buffer_size(1000)
            groupId = ad.declare_group("CartFieldInterp", "")
            ad.select_method(groupId, "POSIX1", "", "")
            #.Define variables and attributes.
            ad.define_attribute_byvalue(groupId, "numCells", "", nx)
            lo = np.zeros(np.size(nx), dtype='double')
            up = np.zeros(np.size(nx), dtype='double')
            for i in range(np.size(nx)):
                lo[i], up[i] = x[i][0], x[i][-1]
            ad.define_attribute_byvalue(groupId, "lowerBounds", "", lo)
            ad.define_attribute_byvalue(groupId, "upperBounds", "", up)
            ad.define_var(groupId, "CartGridFieldInterpTimeAv", "",
                          ad.DATATYPE.double, sNumCells, sNumCells, sOffsets)
            fh = ad.open("CartFieldInterp", tAvFile, 'w')
            ad.write(fh, "CartGridFieldInterpTimeAv", q0AvT)
            ad.close(fh)
            ad.finalize()
            #.Deal with weird file output where a '.bp.0' file is created.
            if len(tAvFile.split('/')) > 1:
                nm = tAvFile.split('/')[-1]
            else:
                nm = tAvFile
            shutil.move(tAvFile + '.dir/' + nm + '.0', tAvFile)
            shutil.rmtree(tAvFile + '.dir')
    else:
        #.Read time average from existent file.
        print(" ")
        print("Reading time average in " + tAvFile + " ...")
        hF = ad.file(tAvFile)
        q0AvT = hF['CartGridFieldInterpTimeAv'].read()
        hF.close()

    return q0AvT
Example #10
0
## Writing
print "\n>>> Writing ...\n"

ad.init("config.xml")
fd = ad.open("temperature", "adios_test.bp", "w")

NX = 10
size = 2
groupsize =  4 + 4 + 8 * size * NX
t = np.array(range(NX*size), dtype=np.float64)
tt = t.reshape((size, NX))
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", NX)
ad.write_int(fd, "size", size)
ad.write(fd, "temperature", tt)
ad.close(fd)

ad.finalize()

## Reading
print "\n>>> Reading ...\n"

f = ad.file("adios_test.bp")
f.printself()

v = f.var['temperature']
v.printself()

val = v.read()
print val
Example #11
0
#!/usr/bin/env python
import adios
import numpy as np

adios.init("config.xml")
fd = adios.open("temperature", "adios_test.bp", "w")

NX = 10
size = 1
rank = 0
groupsize =  4 + 4 + 4 + 8 * 1 * NX
t = np.array(range(NX), dtype=np.float64)
adios.set_group_size(fd, groupsize)
adios.write_int(fd, "NX", NX)
adios.write_int(fd, "size", size)
adios.write_int(fd, "rank", rank)
adios.write(fd, "temperature", t)
adios.close(fd)

adios.finalize()

Example #12
0
    def write(self, bufferSize=1000, outName=None, mode='bp'):
        """Writes data in ADIOS .bp file, ASCII .txt file, or NumPy .npy file
        """
        # Create output file name
        if outName is None:
            if self.fileName is not None:
                fn = self.fileName
                outName = fn.split('.')[0].strip('_') + '_mod.' + mode
            else:
                outName = "gdata." + mode
            #end
        else:
            if not isinstance(outName, str):
                raise TypeError("'outName' must be a string")
            #end
            if outName.split('.')[-1] != mode:
                outName += '.' + mode
            #end
        #end

        numDims = self.getNumDims()
        numComps = self.getNumComps()
        numCells = self.getNumCells()

        if mode == 'bp':
            # Create string number of cells and offsets
            sNumCells = ""
            sOffsets = ""
            for i in range(numDims):
                sNumCells += "{:d},".format(int(numCells[i]))
                sOffsets += "0,"
            #end
            sNumCells += "{:d}".format(numComps)
            sOffsets += "0"

            # ADIOS init
            adios.init_noxml()
            adios.set_max_buffer_size(bufferSize)
            groupId = adios.declare_group("CartField", "")
            adios.select_method(groupId, "POSIX1", "", "")

            # Define variables and attributes
            adios.define_attribute_byvalue(groupId, "numCells", "", numCells)
            lo, up = self.getBounds()
            adios.define_attribute_byvalue(groupId, "lowerBounds", "", lo)
            adios.define_attribute_byvalue(groupId, "upperBounds", "", up)
            fh = adios.open("CartField", outName, 'w')

            if self.meta["time"]:
                adios.define_var(groupId, "time", "", adios.DATATYPE.double,
                                 "", "", "")
                adios.write(fh, key, self.meta["time"])
            #end

            adios.define_var(groupId, self._varName, "", adios.DATATYPE.double,
                             sNumCells, sNumCells, sOffsets)

            adios.write(fh, self._varName, self.getValues())
            adios.close(fh)
            adios.finalize()

            # Cheating
            if len(outName.split('/')) > 1:
                nm = outName.split('/')[-1]
            else:
                nm = outName
            #end
            shutil.move(outName + '.dir/' + nm + '.0', outName)
            shutil.rmtree(outName + '.dir')
        elif mode == 'txt':
            numRows = int(numCells.prod())
            grid = self.getGrid()
            values = self.getValues()

            basis = np.full(numDims, 1.0)
            for d in range(numDims - 1):
                basis[d] = numCells[(d + 1):].prod()
            #end

            fh = open(outName, 'w')
            for i in range(numRows):
                idx = i
                idxs = np.zeros(numDims, np.int)
                for d in range(numDims):
                    idxs[d] = int(idx // basis[d])
                    idx = idx % basis[d]
                #end
                line = ""
                for d in range(numDims - 1):
                    line += "{:.15e}, ".format(grid[d][idxs[d]])
                #end
                line += "{:.15e}, ".format(grid[numDims - 1][idxs[numDims -
                                                                  1]])
                for c in range(numComps - 1):
                    line += "{:.15e}, ".format(values[tuple(idxs)][c])
                #end
                line += "{:.15e}\n".format(values[tuple(idxs)][numComps - 1])
                fh.write(line)
            #end
            fh.close()
        elif mode == 'npy':
            values = self.getValues()

            np.save(outName, values.squeeze())