def setUp(self):
NX = 10
size = 2
t = np.array(range(NX*size), dtype=np.float64)
tt = t.reshape((size, NX))
self.temp = TempFile()
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
fw = ad.writer(self.temp.path)
fw.declare_group('group', method='POSIX1')
fw['NX'] = NX
fw['size'] = size
fw['temperature'] = tt
fw.attrs['/temperature/description'] = "Global array written from 'size' processes"
fw.attrs["/someSubGroup/anOtherGroup/anOtherAttribute"] = 99
fw["/someSubGroup/anOtherGroup/anOtherVariable"] = 77
fw.close()
ad.finalize()
self.f = ad.file(self.temp.path)
def setUp(self):
self.temp = TempFile()
ad.init_noxml()
g = ad.declare_group("temperature", "", ad.FLAG.YES)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
"size,NX", "0,0")
self.msg = "this is a test"
ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
ad.select_method(g, "POSIX1", "verbose=3", "")
for i in range(5):
mode = "a"
if i == 0: mode = "w"
fd = ad.open("temperature", self.temp.path, mode)
self.NX = 10
self.size = 2
##groupsize = 4 + 4 + 8 * self.size * self.NX
t = np.array(list(range(self.NX * self.size)),
dtype=np.float64) + 100 * i
self.tt = t.reshape((self.size, self.NX))
##ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", self.NX)
ad.write_int(fd, "size", self.size)
ad.write(fd, "temperature", self.tt)
ad.close(fd)
ad.finalize()
self.f = ad.file(self.temp.path)
def setUp(self):
self.temp = TempFile()
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
g = ad.declare_group("temperature", "", ad.FLAG.YES)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
"size,NX", "0,0")
self.msg = "this is a test"
self.unit = "C"
## attr must be <varpath> + / + something without / (line 857, common_read.c)
ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string,
self.msg, "")
ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string,
self.unit, "")
ad.select_method(g, "POSIX1", "verbose=3", "")
fd = ad.open("temperature", self.temp.path, "w")
self.NX = 10
self.size = 2
groupsize = 4 + 4 + 8 * self.size * self.NX
t = np.array(range(self.NX * self.size), dtype=np.float64)
self.tt = t.reshape((self.size, self.NX))
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", self.NX)
ad.write_int(fd, "size", self.size)
ad.write(fd, "temperature", self.tt)
ad.close(fd)
ad.finalize()
self.f = ad.file(self.temp.path)
def setUp(self):
self.temp = TempFile()
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
g = ad.declare_group("temperature", "", ad.FLAG.YES)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
self.msg = "this is a test"
ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, "C", "")
ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string, "description", "")
ad.define_attribute(g, "/subgroup/subsubgroup/otherattr", "", ad.DATATYPE.string, "another", "")
ad.define_var(g, "/subgroup/subsubgroup/othervar", "", ad.DATATYPE.integer, "", "", "")
ad.select_method(g, "POSIX1", "verbose=3", "")
fd = ad.open("temperature", self.temp.path, "w")
self.NX = 10
self.size = 2
groupsize = 4 + 4 + 8 * self.size * self.NX + 4
t = np.array(range(self.NX * self.size), dtype=np.float64)
self.tt = t.reshape((self.size, self.NX))
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", self.NX)
ad.write_int(fd, "size", self.size)
ad.write(fd, "temperature", self.tt)
ad.write_int(fd, "/subgroup/subsubgroup/othervar", 99)
ad.close(fd)
ad.finalize()
self.f = ad.file(self.temp.path)
def setUp(self):
self.temp = TempFile()
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
g = ad.declare_group("temperature", "", ad.FLAG.YES)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
self.msg = "this is a test"
self.unit = "C"
## attr must be <varpath> + / + something without / (line 857, common_read.c)
ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string, self.msg, "")
ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, self.unit, "")
ad.select_method(g, "POSIX1", "verbose=3", "")
fd = ad.open("temperature", self.temp.path, "w")
self.NX = 10
self.size = 2
groupsize = 4 + 4 + 8 * self.size * self.NX
t = np.array(range(self.NX * self.size), dtype=np.float64)
self.tt = t.reshape((self.size, self.NX))
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", self.NX)
ad.write_int(fd, "size", self.size)
ad.write(fd, "temperature", self.tt)
ad.close(fd)
ad.finalize()
self.f = ad.file(self.temp.path)
def setUp(self):
self.temp = TempFile()
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
g = ad.declare_group("temperature", "", ad.FLAG.YES)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
"size,NX", "0,0")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX",
"size,NX", "0,0")
self.msg = "this is a test"
ad.define_attribute(g, "desc", "", ad.DATATYPE.string, self.msg, "")
ad.define_attribute(g, "temperature/unit", "", ad.DATATYPE.string, "C",
"")
ad.define_attribute(g, "temperature/desc", "", ad.DATATYPE.string,
"description", "")
ad.define_attribute(g, "/subgroup/subsubgroup/otherattr", "",
ad.DATATYPE.string, "another", "")
ad.define_var(g, "/subgroup/subsubgroup/othervar", "",
ad.DATATYPE.integer, "", "", "")
ad.select_method(g, "POSIX1", "verbose=3", "")
fd = ad.open("temperature", self.temp.path, "w")
self.NX = 10
self.size = 2
groupsize = 4 + 4 + 8 * self.size * self.NX + 4
t = np.array(range(self.NX * self.size), dtype=np.float64)
self.tt = t.reshape((self.size, self.NX))
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", self.NX)
ad.write_int(fd, "size", self.size)
ad.write(fd, "temperature", self.tt)
ad.write_int(fd, "/subgroup/subsubgroup/othervar", 99)
ad.close(fd)
ad.finalize()
self.f = ad.file(self.temp.path)
def write_scalar(self, adtype, val, varname='val'):
ad.init_noxml()
g = ad.declare_group("group", "", ad.FLAG.YES)
ad.define_var(g, varname, "", adtype, "", "", "")
ad.select_method(g, "POSIX1", "", "")
if adtype == ad.DATATYPE.string:
dtype = ad.adios2npdtype(adtype, len(str(val)))
npval = np.array(val, dtype=dtype)
else:
dtype = ad.adios2npdtype(adtype)
npval = np.array(val, dtype=dtype)
fd = ad.open("group", self.temp.path, "w")
##ad.set_group_size(fd, npval.nbytes)
ad.write(fd, varname, val, dtype)
ad.close(fd)
ad.finalize()
f = ad.file(self.temp.path)
v = f.vars['val']
self.assertEqual(v.read(), npval)
def write(self, msg):
doid = msg['doid']
column = msg['column']
rows = msg['rows']
shape = msg['shape']
rowid = msg['row']
datatype = msg['datatype']
data = msg['data']
filename = self.filepath + '/' + doid
dimension = [rows] + copy.copy(shape)
global_dimension = copy.copy(dimension)
global_dimension[0] = self.maxrows
offset = [0] * len(dimension)
offset[0] = rowid
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, self.buffersize)
adios_group = ad.declare_group("shore", "", ad.FLAG.YES)
local_dimension = str(dimension)[1:-1]
global_dimension = str(global_dimension)[1:-1]
offset = str(offset)[1:-1]
ad.select_method(adios_group, "POSIX", "", "")
ad.define_var(adios_group, column, "",
self.dtype_numpy2adios[datatype], local_dimension,
global_dimension, offset)
adios_file = ad.open("shore", filename, 'u')
group_size = 1000000
if data.nbytes > group_size:
group_size = data.nbytes
ad.set_group_size(adios_file, group_size)
ad.write(adios_file, column, data, datatype)
ad.close(adios_file)
ad.finalize()
return True
def write_scalar(self, adtype, val, varname='val'):
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
g = ad.declare_group("group", "", ad.FLAG.YES)
ad.define_var(g, varname, "", adtype, "", "", "")
ad.select_method(g, "POSIX1", "", "")
if adtype == ad.DATATYPE.string:
dtype = ad.adios2npdtype(adtype, len(str(val)))
npval = np.array(val, dtype=dtype)
else:
dtype = ad.adios2npdtype(adtype)
npval = np.array(val, dtype=dtype)
fd = ad.open("group", self.temp.path, "w")
ad.set_group_size(fd, npval.nbytes)
ad.write(fd, varname, val, dtype)
ad.close(fd)
ad.finalize()
f = ad.file(self.temp.path)
v = f.vars['val']
self.assertEqual(v.read(), npval)
def getTimeAv(dataDir, simName, varName, iFrame, fFrame, p, b, saveAv, tAvDir):
#.Check or create post data directory.
checkMkdir(tAvDir)
#.Check if time average file already exists.
tAvFile = tAvDir + simName + '_' + varName + '_TimeAv' + str(
iFrame) + '-' + str(fFrame) + '.bp'
if not os.path.isfile(tAvFile):
#.Compute time average and store it in new file.
fileName = dataDir + simName + '_' + varName + '_%d.bp'
x, gridDim, nx, lx, dx = getGrid(fileName % iFrame,
p,
b,
location='center')
q0AvT = np.zeros(nx)
for nFr in range(iFrame, fFrame + 1):
#.Read 3D data into q0.
q0AvT = np.add(q0AvT,
np.squeeze(getInterpData(fileName % nFr, p, b)))
q0AvT = np.divide(q0AvT, float(fFrame - iFrame + 1))
if saveAv:
#.Save time average to a file for reuse.
print(" ")
print("Saving time average in " + tAvFile + " ...")
#.Function to write DG coefficients to Gkeyll-style ADIOS file.
sNumCells = ""
sOffsets = ""
for i in range(np.size(nx)):
sNumCells += "{:d},".format(int(nx[i]))
sOffsets += "0,"
#.ADIOS init.
ad.init_noxml()
ad.set_max_buffer_size(1000)
groupId = ad.declare_group("CartFieldInterp", "")
ad.select_method(groupId, "POSIX1", "", "")
#.Define variables and attributes.
ad.define_attribute_byvalue(groupId, "numCells", "", nx)
lo = np.zeros(np.size(nx), dtype='double')
up = np.zeros(np.size(nx), dtype='double')
for i in range(np.size(nx)):
lo[i], up[i] = x[i][0], x[i][-1]
ad.define_attribute_byvalue(groupId, "lowerBounds", "", lo)
ad.define_attribute_byvalue(groupId, "upperBounds", "", up)
ad.define_var(groupId, "CartGridFieldInterpTimeAv", "",
ad.DATATYPE.double, sNumCells, sNumCells, sOffsets)
fh = ad.open("CartFieldInterp", tAvFile, 'w')
ad.write(fh, "CartGridFieldInterpTimeAv", q0AvT)
ad.close(fh)
ad.finalize()
#.Deal with weird file output where a '.bp.0' file is created.
if len(tAvFile.split('/')) > 1:
nm = tAvFile.split('/')[-1]
else:
nm = tAvFile
shutil.move(tAvFile + '.dir/' + nm + '.0', tAvFile)
shutil.rmtree(tAvFile + '.dir')
else:
#.Read time average from existent file.
print(" ")
print("Reading time average in " + tAvFile + " ...")
hF = ad.file(tAvFile)
q0AvT = hF['CartGridFieldInterpTimeAv'].read()
hF.close()
return q0AvT
import adios as ad
import numpy as np
import getopt, sys
import os
method = "POSIX"
init = "verbose=3;"
if len(sys.argv) > 1:
method = sys.argv[1]
if len(sys.argv) > 2:
init = sys.argv[2]
## Init
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
g = ad.declare_group("temperature", "", 1)
ad.define_var(g, "NX", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "size", "", ad.DATATYPE.integer, "", "", "")
ad.define_var(g, "temperature", "", ad.DATATYPE.double, "size,NX", "size,NX", "0,0")
ad.select_method(g, method, init, "")
print ">>> Method:", method
## Writing
for i in range(5):
fd = ad.open("temperature", "temp.bp", "a")
NX = 10
size = 2
groupsize = 4 + 4 + 8 * size * NX
$ python ./test_adios_write.py
"""
import adios as ad
import numpy as np
print("\n>>> Prepare ...\n")
fname = 'adios_test_writer.bp'
NX = 10
size = 2
t = np.array(list(range(NX * size)), dtype=np.float64)
tt = t.reshape((size, NX))
print("\n>>> Writing ...\n")
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
fw = ad.writer(fname)
fw.declare_group('group', method='POSIX1')
fw['NX'] = NX
fw['size'] = size
fw['temperature'] = tt
fw.attrs[
'/temperature/description'] = "Global array written from 'size' processes"
fw.close()
## Reading
print("\n>>> Reading ...\n")
def setUp(self):
ad.init_noxml()
def setUp(self):
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
def setUp(self):
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
def write(self, bufferSize=1000, outName=None, mode='bp'):
"""Writes data in ADIOS .bp file, ASCII .txt file, or NumPy .npy file
"""
# Create output file name
if outName is None:
if self.fileName is not None:
fn = self.fileName
outName = fn.split('.')[0].strip('_') + '_mod.' + mode
else:
outName = "gdata." + mode
#end
else:
if not isinstance(outName, str):
raise TypeError("'outName' must be a string")
#end
if outName.split('.')[-1] != mode:
outName += '.' + mode
#end
#end
numDims = self.getNumDims()
numComps = self.getNumComps()
numCells = self.getNumCells()
if mode == 'bp':
# Create string number of cells and offsets
sNumCells = ""
sOffsets = ""
for i in range(numDims):
sNumCells += "{:d},".format(int(numCells[i]))
sOffsets += "0,"
#end
sNumCells += "{:d}".format(numComps)
sOffsets += "0"
# ADIOS init
adios.init_noxml()
adios.set_max_buffer_size(bufferSize)
groupId = adios.declare_group("CartField", "")
adios.select_method(groupId, "POSIX1", "", "")
# Define variables and attributes
adios.define_attribute_byvalue(groupId, "numCells", "", numCells)
lo, up = self.getBounds()
adios.define_attribute_byvalue(groupId, "lowerBounds", "", lo)
adios.define_attribute_byvalue(groupId, "upperBounds", "", up)
fh = adios.open("CartField", outName, 'w')
if self.meta["time"]:
adios.define_var(groupId, "time", "", adios.DATATYPE.double,
"", "", "")
adios.write(fh, key, self.meta["time"])
#end
adios.define_var(groupId, self._varName, "", adios.DATATYPE.double,
sNumCells, sNumCells, sOffsets)
adios.write(fh, self._varName, self.getValues())
adios.close(fh)
adios.finalize()
# Cheating
if len(outName.split('/')) > 1:
nm = outName.split('/')[-1]
else:
nm = outName
#end
shutil.move(outName + '.dir/' + nm + '.0', outName)
shutil.rmtree(outName + '.dir')
elif mode == 'txt':
numRows = int(numCells.prod())
grid = self.getGrid()
values = self.getValues()
basis = np.full(numDims, 1.0)
for d in range(numDims - 1):
basis[d] = numCells[(d + 1):].prod()
#end
fh = open(outName, 'w')
for i in range(numRows):
idx = i
idxs = np.zeros(numDims, np.int)
for d in range(numDims):
idxs[d] = int(idx // basis[d])
idx = idx % basis[d]
#end
line = ""
for d in range(numDims - 1):
line += "{:.15e}, ".format(grid[d][idxs[d]])
#end
line += "{:.15e}, ".format(grid[numDims - 1][idxs[numDims -
1]])
for c in range(numComps - 1):
line += "{:.15e}, ".format(values[tuple(idxs)][c])
#end
line += "{:.15e}\n".format(values[tuple(idxs)][numComps - 1])
fh.write(line)
#end
fh.close()
elif mode == 'npy':
values = self.getValues()
np.save(outName, values.squeeze())