config = "config_mpi.xml"
if len(sys.argv) > 1:
config = sys.argv[1]
ad.init(config, comm)
fd = ad.open("temperature", "adios_test_mpi.bp", "w", comm)
NX = 10
groupsize = 4 + 4 + 4 + 8 * 1 * NX
t = np.array(range(NX), dtype=np.float64) + rank*NX
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", NX)
ad.write_int(fd, "rank", rank)
ad.write_int(fd, "size", size)
ad.write(fd, "temperature", t)
ad.close(fd)
ad.finalize()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file("adios_test_mpi.bp", comm=MPI.COMM_SELF)
f.printself()
v = f.var['temperature']
v.printself()
val = v.read()
print val
def sosToADIOS():
global SOS
global config
parseConfigFile()
SOS = SSOS()
printf("Initializing SOS: ...\b\b\b")
SOS.init()
printf("OK!\n")
#####
#
# Get the maximum simulation cycle found in the database.
#
# NOTE: The cycleFieldName variable should match what is being used
# either by your application or SOSflow. If you are not using
# an explicit cycle value, you can use SOSflow's internal
# field named "frame" that is updated every time SOS_publish(...)
# is called. As long as you are publishing to SOS at the end
# of major program steps, this will give you what you want.
#
# NOTE: For online queries, if you want to ensure that your most
# current projection represents a complete set of values,
# and you're investigating a block-synchronous code, you can
# grab the current maximum and subtract one.
#
#
num_rows = 0
# Get at least one active aggregator
lookupAggregators()
g = None
if config["output_adios"]:
# ADIOS file output
ad.init_noxml()
g = ad.declare_group("TAU_metrics", "", ad.FLAG.YES)
ad.define_var(g, "program_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "comm_rank_count", "", ad.DATATYPE.unsigned_integer,
"", "", "")
ad.define_var(g, "thread_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "metric_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "timer_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "timer_value_count", "", ad.DATATYPE.unsigned_integer,
"", "", "")
ad.define_var(g, "timer_values", "", ad.DATATYPE.unsigned_integer,
"timer_value_count,6", "timer_value_count,6", "0,0")
ad.define_var(g, "counter_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "counter_value_count", "",
ad.DATATYPE.unsigned_integer, "", "", "")
ad.define_var(g, "counter_values", "", ad.DATATYPE.double,
"counter_value_count,5", "counter_value_count,5", "0,0")
print "using ADIOS method:", str(config["adios_method"])
ad.select_method(g, str(config["adios_method"]), "verbose=3", "")
# wait for a frame to show up. Frame 0 (and maybe 1) are TAU metadata.
# The rest should be just timers.
next_frame = 0
# first iteration, we are writing the file. after that, appending.
adios_mode = "w"
# Keep running until there are no more frames to wait for.
# At runtime, this is a moving target, since next_frame gets updated.
while config["aggregators"][
"runtime"] or next_frame < config["aggregators"]["maxframe"]:
# wait for the next batch of frames
timeout = waitForServer(SOS, next_frame)
if timeout:
break
print "Processing frame", next_frame
start = time.time()
fd = ad.open("TAU_metrics", "tau-metrics.bp", adios_mode)
writeMetaData(SOS, next_frame, g, fd)
writeTimerData(SOS, next_frame, g, fd)
writeCounterData(SOS, next_frame, g, fd)
ad.close(fd)
# future iterations are appending, not writing
adios_mode = "a"
# clean up the database for long runs
cleanDB(SOS, next_frame)
next_frame = next_frame + 1
end = time.time()
print "loop time:", str(end - start)
# finalize adios
if config["output_adios"]:
ad.finalize()
# finalize SOS
SOS.finalize()
print " ...DONE!"
return
def generateADIOSFile(SOS, cycleFieldName, simCycle, lastX, lastY, lastZ,
stride, mintime, maxtime):
global previous_attr
global cached_results_dim
global cachedName
global config
global last_time
global last_fp_ops
global last_rss
global adios_mode
# Get the frame-specific data...
# because we are querying different servers, the data has the potential to
# arrive out-of-order (prog_name, comm_rank). So after each query, sort
# the results into a dictionary of dictionaries.
prog_names = {}
# do the memory first - HWM
start = time.time()
sqlValsToColByRank = "select coalesce(value,0.0), prog_name, comm_rank from viewCombined where value_name = '" + str(
config["events"]["counters"][0]) + "' and frame = " + str(
simCycle) + ";"
results, col_names = queryAllAggregators(sqlValsToColByRank)
end = time.time()
print(end - start), "seconds for frame query"
for r in results:
value = float(r[0])
prog_name = str(r[1])
comm_rank = int(r[2])
if prog_name not in prog_names:
prog_names[prog_name] = {}
if comm_rank not in prog_names[prog_name]:
prog_names[prog_name][comm_rank] = []
prog_names[prog_name][comm_rank].append(value)
# do the memory first - RSS
start = time.time()
current_rss = {}
sqlValsToColByRank = "select coalesce(value,0.0), prog_name, comm_rank from viewCombined where value_name = '" + str(
config["events"]["counters"][1]) + "' and frame = " + str(
simCycle) + ";"
results, col_names = queryAllAggregators(sqlValsToColByRank)
end = time.time()
print(end - start), "seconds for frame query"
index = -1
for r in results:
index = index + 1
value = float(r[0])
# change the mean value to a total value and save it
current_rss[index] = value * simCycle
# convert the running average to the most recent measurement
if index in last_rss:
value = current_rss[index] - last_rss[index]
prog_name = str(r[1])
comm_rank = int(r[2])
if prog_name not in prog_names:
prog_names[prog_name] = {}
if comm_rank not in prog_names[prog_name]:
prog_names[prog_name][comm_rank] = []
prog_names[prog_name][comm_rank].append(value)
last_rss = current_rss
# do the timer PAPI metric next
start = time.time()
# Make sure we sort by prog_name, comm_rank!
sqlValsToColByRank = "select coalesce(value,0.0), prog_name, comm_rank from viewCombined where value_name = '" + str(
config["events"]["timers"][0]) + "' and frame = " + str(simCycle) + ";"
results, col_names = queryAllAggregators(sqlValsToColByRank)
end = time.time()
print(end - start), "seconds for frame query"
for r in results:
value = float(r[0])
prog_name = str(r[1])
comm_rank = int(r[2])
if prog_name not in prog_names:
prog_names[prog_name] = {}
if comm_rank not in prog_names[prog_name]:
prog_names[prog_name][comm_rank] = []
prog_names[prog_name][comm_rank].append(value)
# do the timer time metric next
start = time.time()
sqlValsToColByRank = "select coalesce(value,0.0), prog_name, comm_rank from viewCombined where value_name = '" + str(
config["events"]["timers"][1]) + "' and frame = " + str(simCycle) + ";"
results, col_names = queryAllAggregators(sqlValsToColByRank)
end = time.time()
print(end - start), "seconds for frame query"
for r in results:
value = float(r[0])
prog_name = str(r[1])
comm_rank = int(r[2])
if prog_name not in prog_names:
prog_names[prog_name] = {}
if comm_rank not in prog_names[prog_name]:
prog_names[prog_name][comm_rank] = []
prog_names[prog_name][comm_rank].append(value)
# now that the data is queried and sorted, write it out to the file
# initialize the ADIOS data
groupsize = 0
# How many MPI ranks in each program?
for prog_name in prog_names:
groupsize = groupsize + len(prog_names[prog_name])
program_count = len(prog_names)
adios_process_index = np.array(range(groupsize), dtype=np.uint32)
adios_memdata_hwm = np.array(range(groupsize), dtype=np.float64)
adios_memdata_rss = np.array(range(groupsize), dtype=np.float64)
adios_flops1 = np.array(range(groupsize), dtype=np.float64)
adios_flops2 = np.array(range(groupsize), dtype=np.float64)
adios_program = np.array(range(groupsize), dtype=np.uint32)
adios_program_name = np.array(range(program_count), dtype=np.chararray)
adios_mpi_index = np.array(range(groupsize), dtype=np.uint32)
cyclestr = str(simCycle)
cyclestr = cyclestr.rjust(5, '0')
filename = "performance.metrics." + cyclestr + ".txt"
# regular text file
if config["output_text"]:
flops_out = open(config["outputdir"] + "/" + filename, 'w')
flops_out.write(
"Process Index, Memory HWM, Memory RSS, Total FLOPS, Latest FLOPS, Program Name, Program Index, MPI Rank\n"
)
s1 = config["events"]["timer_scaling"][0]
s2 = config["events"]["timer_scaling"][1]
index = -1
prog_index = -1
current_time = {}
current_fp_ops = {}
for prog_name in prog_names:
prog_index = prog_index + 1
adios_program_name[prog_index] = prog_name
for comm_rank in prog_names[prog_name]:
index = index + 1
print prog_name, comm_rank, index, current_fp_ops, prog_names
current_fp_ops[index] = prog_names[prog_name][comm_rank][2] * s1
current_time[index] = prog_names[prog_name][comm_rank][3] * s2
flops_to_date = current_fp_ops[index] / current_time[index]
if len(last_fp_ops) > 0:
tmp = (current_fp_ops[index] - last_fp_ops[index])
tmp2 = (current_time[index] - last_time[index])
if tmp2 > 0.0:
# compute flops from lastest timestep
flops_in_last_timestep = tmp / tmp2
else:
if last_time[index] > 0.0:
# compute flops from previous timestep
flops_in_last_timestep = last_fp_ops[
index] / last_time[index]
else:
# something weird is happening...
flops_in_last_timestep = 0.0
else:
# compute flops from first timestep
flops_in_last_timestep = flops_to_date
if config["output_text"]:
# write the sorted data to the file
flops_out.write(str(index) + ", ")
flops_out.write(
str(prog_names[prog_name][comm_rank][0]) + ", ")
flops_out.write(
str(prog_names[prog_name][comm_rank][1]) + ", ")
flops_out.write(str(flops_to_date) + ", ")
flops_out.write(str(flops_in_last_timestep) + ", ")
flops_out.write(prog_name + ", ")
flops_out.write(str(prog_index) + ", ")
flops_out.write(str(comm_rank) + "\n")
if config["output_adios"]:
adios_process_index[index] = index
adios_memdata_hwm[index] = prog_names[prog_name][comm_rank][0]
adios_memdata_rss[index] = prog_names[prog_name][comm_rank][1]
adios_flops1[index] = flops_to_date
adios_flops2[index] = flops_in_last_timestep
adios_program[index] = prog_index
adios_mpi_index[index] = comm_rank
if config["output_text"]:
flops_out.close()
stream_file = "performance.metrics.txt"
stream_out = open(config["outputdir"] + "/" + stream_file, 'w')
stream_out.write(filename + "\n")
stream_out.close()
last_time = current_time
last_fp_ops = current_fp_ops
if config["output_adios"]:
# write the adios
fd = ad.open("TAU_metrics", "tau-metrics.bp", adios_mode)
ad.write_int(fd, "process_count", groupsize)
ad.write_int(fd, "program_count", program_count)
ad.write(fd, "process_index", adios_process_index)
ad.write(fd, "memory_HWM", adios_memdata_hwm)
ad.write(fd, "memory_RSS", adios_memdata_rss)
ad.write(fd, "total_FLOPS", adios_flops1)
ad.write(fd, "latest_FLOPS", adios_flops2)
#ad.write(fd, "program_name", adios_program_name)
ad.write(fd, "program_index", adios_program)
ad.write(fd, "MPI_rank", adios_mpi_index)
ad.close(fd)
#fd.close()
# future iterations are appending, not writing
adios_mode = "a"
return filename
def sosToADIOS():
global SOS
global config
global validation
parseConfigFile()
SOS = SSOS()
printf("Initializing SOS: ...\b\b\b")
SOS.init()
printf("OK!\n")
#####
#
# Get the maximum simulation cycle found in the database.
#
# NOTE: The cycleFieldName variable should match what is being used
# either by your application or SOSflow. If you are not using
# an explicit cycle value, you can use SOSflow's internal
# field named "frame" that is updated every time SOS_publish(...)
# is called. As long as you are publishing to SOS at the end
# of major program steps, this will give you what you want.
#
# NOTE: For online queries, if you want to ensure that your most
# current projection represents a complete set of values,
# and you're investigating a block-synchronous code, you can
# grab the current maximum and subtract one.
#
#
num_rows = 0
# Get at least one active aggregator
lookupAggregators()
g = None
if config["output_adios"]:
# ADIOS file output
ad.init_noxml()
g = ad.declare_group("TAU_metrics", "", ad.FLAG.YES)
ad.define_var(g, "program_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "comm_rank_count", "", ad.DATATYPE.unsigned_integer,
"", "", "")
ad.define_var(g, "thread_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "event_type_count", "", ad.DATATYPE.unsigned_integer,
"", "", "")
ad.define_var(g, "timer_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "timer_event_count", "", ad.DATATYPE.unsigned_integer,
"", "", "")
ad.define_var(g, "event_timestamps", "", ad.DATATYPE.unsigned_long,
"timer_event_count,6", "timer_event_count,6", "0,0")
ad.define_var(g, "counter_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "counter_event_count", "",
ad.DATATYPE.unsigned_integer, "", "", "")
ad.define_var(g, "counter_values", "", ad.DATATYPE.unsigned_long,
"counter_event_count,6", "counter_event_count,6", "0,0")
ad.define_var(g, "comm_count", "", ad.DATATYPE.unsigned_integer, "",
"", "")
ad.define_var(g, "comm_timestamps", "", ad.DATATYPE.unsigned_long,
"comm_count,8", "comm_count,8", "0,0")
print("using ADIOS method:", str(config["adios_method"]))
ad.select_method(g, str(config["adios_method"]), "verbose=3", "")
# wait for a frame to show up. Frame 0 (and maybe 1) are TAU metadata.
# The rest should be just timers.
next_frame = 0
# first iteration, we are writing the file. after that, appending.
adios_mode = "w"
waitForServer(SOS, 0)
buildColumnMap(SOS)
# Keep running until there are no more frames to wait for.
# At runtime, this is a moving target, since next_frame gets updated.
done = False
total_count = 0
while (not done or total_count > 0) and (
config["aggregators"]["runtime"]
or next_frame < config["aggregators"]["maxframe"]):
# wait for the next batch of frames
if not done:
timeout = waitForServer(SOS, next_frame + 1)
if timeout:
done = True
#if len(column_map) == 0:
# buildColumnMap(SOS)
print("Processing frame", next_frame)
start = time.time()
fd = ad.open("TAU_metrics", "tau-metrics.bp", adios_mode)
meta_count = writeMetaData(SOS, next_frame, g, fd)
timer_count = writeTimerData(SOS, next_frame, g, fd)
total_count = meta_count + timer_count
ad.close(fd)
# future iterations are appending, not writing
adios_mode = "a"
print("Processed", total_count, "rows")
if total_count == 0 and done:
break
next_frame = next_frame + 1
end = time.time()
print("loop time:", str(end - start))
# finalize adios
if config["output_adios"]:
ad.finalize()
# finalize SOS
SOS.finalize()
for p in validation:
for r in validation[p]:
for t in validation[p][r]:
if len(validation[p][r][t]) != 0:
print("VALIDATION ERROR!", p, r, t, validation[p][r][t],
"was not exited")
print(" ...DONE!")
return
config = "config_mpi.xml"
if len(sys.argv) > 1:
config = sys.argv[1]
ad.init(config, comm)
fd = ad.open("temperature", "adios_test_mpi.bp", "w", comm)
NX = 10
groupsize = 4 + 4 + 4 + 8 * 1 * NX
t = np.array(range(NX), dtype=np.float64) + rank*NX
ad.set_group_size(fd, groupsize)
ad.write_int(fd, "NX", NX)
ad.write_int(fd, "rank", rank)
ad.write_int(fd, "size", size)
ad.write(fd, "temperature", t)
ad.close(fd)
ad.finalize()
## Reading
if rank == 0:
print "\n>>> Reading ...\n"
f = ad.file("adios_test_mpi.bp", comm=MPI.COMM_SELF)
f.printself()
v = f.vars['temperature']
v.printself()
val = v.read()
print val