def __init__(self, configFile):
self.config = configparser.ConfigParser()
self.config.read(configFile)
self.fileType = self.config['Parser']['FileType']
self.inputFile = self.config['Parser']['InputFile'] # input file path
self.parseMode = self.config['Parser']['ParseMode']
self.stream = None
self.status = None
self.bpAttrib = None
self.bpNumAttrib = None
self.numFun = 0
self.funMap = None
self.eventType = None
if self.fileType == "bp":
ad.read_init(
"DATASPACES",
parameters="verbose=3") # initialize adios streaming mode
if self.parseMode == "stream":
self.stream = ad.file(self.inputFile,
"DATASPACES",
is_stream=True,
timeout_sec=-1.0)
self.bpAttrib = self.stream.attr
self.bpNumAttrib = self.stream.nattrs
with open('funMap.pickle', 'rb') as handle:
self.funMap = pickle.load(handle)
handle.close()
with open('eventType.pickle', 'rb') as handle:
self.eventType = pickle.load(handle)
handle.close()
#MGY
#for iter in self.bpAttrib: # extract funciton names and ids
# if iter.startswith('timer'):
# self.numFun = self.numFun + 1
# self.funMap[int(iter.split()[1])] = str(self.bpAttrib[iter].value.decode("utf-8")) # if iter is a string "timer 123" separate timer and 123 and assign 123 as integer key to function map and the function name which is stored in self.bpAttrib[iter].value as a value
# if iter.startswith('event_type'):
# self.eventType[int(iter.split()[1])] = str(self.bpAttrib[iter].value.decode("utf-8"))
#MGY
print("\nAdios stream ready... \n\n")
# Debug
#===================================================================
# print("Num attributes: ", self.bpNumAttrib, "\n")
# print("Attribute names: \n", self.bpAttrib, "\n\n\n")
# print("Num functions: ", self.numFun, "\n")
# print("Function map: \n", self.funMap, "\n\n\n" )
#===================================================================
else:
raise Exception("\nInput file format not supported...\n")
groupsize = 4 + 4 + 4 + 8 * 1 * NX
t = np.array(range(NX), dtype=np.float64) + rank*NX
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", NX)
ad.write_int(fd, "rank", rank)
ad.write_int(fd, "size", size)
ad.write(fd, "temperature", t)
ad.close(fd)
ad.finalize()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file("adios_test_mpi.bp", comm=MPI.COMM_SELF)
f.printself()
v = f.var['temperature']
v.printself()
val = v.read()
print val
assert (int(sum(sum(val))) == (size*NX-1)*(size*NX)/2)
f.close()
print "\n>>> Done.\n"
## Testing
if rank == 0:
print "\n>>> Test utility functions ...\n"
gdim = (size, NX)
offset = (rank, 0)
print "\n>>> Writing ... (rank = %d)\n" % rank
ad.init_noxml()
ad.allocate_buffer(ad.BUFFER_ALLOC_WHEN.NOW, 10)
fw = ad.writer(fname, comm=comm)
fw.declare_group('group', method='MPI')
fw.define_var('temperature', ldim=(1, NX), gdim=gdim, offset=offset)
fw['NX'] = NX
fw['size'] = size
fw['temperature'] = t
fw.attrs[
'/temperature/description'] = "Global array written from 'size' processes"
fw.close()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file(fname, comm=MPI.COMM_SELF)
for key, val in f.vars.iteritems():
print key, '=', val.read()
for key, val in f.attrs.iteritems():
print key, '=', val.value
print "\n>>> Done.\n"
gdim = (size, NX)
offset = (rank, 0)
print "\n>>> Writing ... (rank = %d)\n" % rank
ad.init_noxml()
ad.allocate_buffer (ad.BUFFER_ALLOC_WHEN.NOW, 10);
fw = ad.writer(fname, comm=comm)
fw.declare_group('group', method='MPI')
fw.define_var('temperature', ldim=(1,NX), gdim=gdim, offset=offset)
fw['NX'] = NX
fw['size'] = size
fw['temperature'] = t
fw.attr['/temperature/description'] = "Global array written from 'size' processes"
fw.close()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file(fname, comm=MPI.COMM_SELF)
for key, val in f.var.iteritems():
print key, '=', val.read()
for key, val in f.attr.iteritems():
print key, '=', val.value
print "\n>>> Done.\n"
groupsize = 4 + 4 + 4 + 8 * 1 * NX
t = np.array(range(NX), dtype=np.float64) + rank*NX
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", NX)
ad.write_int(fd, "rank", rank)
ad.write_int(fd, "size", size)
ad.write(fd, "temperature", t)
ad.close(fd)
ad.finalize()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file("adios_test_mpi.bp", comm=MPI.COMM_SELF)
f.printself()
v = f.vars['temperature']
v.printself()
val = v.read()
print val
assert (int(np.sum(val)) == (size*NX-1)*(size*NX)/2)
f.close()
print "\n>>> Done.\n"
## Testing
if rank == 0:
print "\n>>> Test utility functions ...\n"
fname = sys.argv[3]
varnames = sys.argv[4:]
assert (size <= N*M)
rank_x = rank % N
rank_y = rank / N
"""
Read step by step
"""
if rank == 0:
print ""
print ">>> Read step by step"
f = ad.file(fname, comm=MPI.COMM_WORLD)
for i in range(f.current_step, f.last_step):
if rank == 0: print ">>> step: %d" % (i)
for vname in varnames:
v = f.var[vname]
if rank == 0: print ">>> name: %s, global dims: %s" % (vname, v.dims)
if len(v.dims) != 2:
if rank == 0: print ">>> Not a 2D array"
continue
count = np.array(v.dims)/np.array((N, M))
offset = count * np.array((rank_x, rank_y))
if rank_x == N-1:
count[0] = v.dims[0] - count[0]*(N-1)
def __init__(self, configFile):
"""This is the constructor for the Parser class.
The parser has two main modes one is the BP, which is equivalent to offline/batch processing and non-BP, which
allows streaming via one of the Adios streaming methods such as DATASPACES or FLEXPATH.
Args:
configFile (string): The configuration files's name.
Returns:
No return value.
"""
# Initialize configparser
self.config = configparser.ConfigParser(interpolation=None)
self.config.read(configFile)
# Initialize logger
self.logFile = self.config['Debug']['LogFile']
self.logFormat = self.config['Debug']['Format']
self.logLevel = self.config['Debug']['LogLevel']
logging.basicConfig(level=self.logLevel, format=self.logFormat, filename=self.logFile)
self.log = logging.getLogger('PARSER')
# Initialize parser mode
self.parseMode = self.config['Parser']['ParseMode']
# Initialize Adios variables
self.Method = self.config['Adios']['Method']
self.Parameters = self.config['Adios']['Parameters']
self.inputFile = self.config['Adios']['InputFile']
self.stream = None
self.status = None
self.bpAttrib = None
self.bpNumAttrib = None
self.numFun = 0
self.funMap = defaultdict(int)
self.eventType = defaultdict(int)
if self.parseMode == "Adios":
ad.read_init(self.Method, parameters=self.Parameters)
self.stream = ad.file(self.inputFile, self.Method, is_stream=True, timeout_sec=-1.0)
# Info
msg = "Adios handle initialized with method: " + self.Method
self.log.info(msg)
if self.Method == "BP":
# This part is needed because the visualization code requires function map and event type before any actual trace data is sent.
self.bpAttrib = self.stream.attr
self.bpNumAttrib = self.stream.nattrs
for iter in self.bpAttrib: # extract function names and ids
if iter.startswith('timer'):
self.numFun = self.numFun + 1
self.funMap[int(iter.split()[1])] = str(self.bpAttrib[iter].value.decode("utf-8")) # if iter is a string "timer 123" separate timer and 123 and assign 123 as integer key to function map and the function name which is stored in self.bpAttrib[iter].value as a value
if iter.startswith('event_type'):
self.eventType[int(iter.split()[1])] = str(self.bpAttrib[iter].value.decode("utf-8"))
# Info
msg = "Adios using BP method..."
self.log.info(msg)
# Debug
msg = "Number of attributes: " + str(self.bpNumAttrib)
self.log.debug(msg)
msg = "Attribute names: \n" + str(self.bpAttrib)
self.log.debug(msg)
msg = "Number of functions: " + str(self.numFun)
self.log.debug(msg)
msg = "Function map: \n" + str(self.funMap)
self.log.debug(msg)
else:
# Info
msg = "Adios using non BP method..."
self.log.info(msg)
else:
msg = "Parse mode not supported..."
self.log.error(msg)
raise Exception(msg)
M = int(sys.argv[2])
fname = sys.argv[3]
varnames = sys.argv[4:]
assert (size <= N * M)
rank_x = rank % N
rank_y = rank / N
"""
Read step by step
"""
if rank == 0:
print ""
print ">>> Read step by step"
f = ad.file(fname, comm=MPI.COMM_WORLD)
for i in range(f.current_step, f.last_step):
if rank == 0: print ">>> step: %d" % (i)
for vname in varnames:
v = f.var[vname]
if rank == 0: print ">>> name: %s, global dims: %s" % (vname, v.dims)
if len(v.dims) != 2:
if rank == 0: print ">>> Not a 2D array"
continue
count = np.array(v.dims) / np.array((N, M))
offset = count * np.array((rank_x, rank_y))
if rank_x == N - 1:
count[0] = v.dims[0] - count[0] * (N - 1)
init = "verbose=3;"
if len(sys.argv) > 1:
method = sys.argv[1]
if len(sys.argv) > 2:
init = sys.argv[2]
print(">>> Method:", method, init)
if method in ["DIMES", "DATASPACES", "FLEXPATH"]:
is_stream = True
else:
is_stream = False
ad.read_init(method, parameters=init)
f = ad.file("temp.bp", method, is_stream=is_stream, timeout_sec=10.0)
print(f)
i = 0
while True:
print(">>> step:", i)
v = f.var['temperature']
print(v)
#val = v.read(nsteps=1)
#val = v[1:2, 1:5]
print(v[...])
if method in ["DIMES", "DATASPACES", "FLEXPATH"]:
print("Advance ...")
if (f.advance() < 0):
import getopt, sys
import os
method = "BP"
init = "verbose=3;"
if len(sys.argv) > 1:
method = sys.argv[1]
if len(sys.argv) > 2:
init = sys.argv[2]
print ">>> Method:", method, init
ad.read_init(method, parameters=init)
f = ad.file("temp.bp", method, is_stream=True, timeout_sec = 10.0)
f.printself()
i = 0
while True:
print ">>> step:", i
v = f.var['temperature']
v.printself()
val = v.read(nsteps=1)
print val
if (f.advance() < 0):
break
i += 1
self.total_samples = self.total_samples + 1
self.bq[self.bq_index] = dataframe_t
self.bq_index = self.bq_index + 1
if self.bq_index == (self.batch_size):
self.adaptive_sampling_update()
self.bq_index = 0
np.set_printoptions(threshold=np.inf)
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
ad.read_init("BP", comm, "verbose=3;")
f = ad.file("nwchem_xyz.bp", "BP", comm, timeout_sec=20.0)
num_steps = f.last_step - f.current_step + 1
vx = f.vars['LX']
vy = f.vars['LY']
vz = f.vars['LZ']
natoms = vx.dims[0]
localatoms = math.ceil(natoms / size)
if rank == (size - 1):
localatoms = natoms - localatoms * (size - 1)
offset = math.ceil(natoms / size) * rank
if rank == 0:
mds2 = MDTrSampler(natoms,
2,