def get_whole_interface_prediction(npart, domain, box, mixing, np, nn, ts,
diff, acc):
fitting_vars = pppm_fit_samples(npart, domain, box, mixing, np, nn, ts,
diff)
grids = get_grid_sizes(npart, domain, box)
# c estimator
f_real_space_prediction = fitting_vars[0]
f_neigh_prediction = fitting_vars[1]
f_kspace_prediction = fitting_vars[2]
estimated_timings = []
for params in get_interface_points(npart, domain, box):
(cutoff_idx, P_idx, grid_idx, ewald_error) = params
## parameters of interest
cutoff = Config.cutoff_values[cutoff_idx]
grid = grids[grid_idx]
n_grid_points = mult(grid)
## estimated time
pt = f_real_space_prediction(cutoff)
nt = f_neigh_prediction(cutoff)
dt = f_kspace_prediction(cutoff, P_idx, n_grid_points)
#tt = pt + dt # c*cutoff**3 + p + g*n_grid_points
tt = pt + nt + dt
# store prediction
estimated_timings.append(((tt, pt, dt, nt), params))
return estimated_timings
def get_error_reciprocal_grid( domain, box, N, C ):
error_grid_file = Config.get_test_kspace_error_grid_file( Config.test )
if not os.path.isfile( error_grid_file ):
print "Error estimates not available, running..."
config_file = Config.get_test_kspace_error_grid_config_file( Config.test )
f = open(config_file, "w")
generate_kspace_error_config_file( Config.test, f )
f.close()
subprocess.check_call("mpirun -np %d C/MPI-qopt %s %s > %s" % \
(Config.test.nprocesses, Config.test.diff_mode, config_file, error_grid_file), shell=True)
print " Done"
#raise Exception # abort until done
grids = get_grid_sizes( N, domain, box )
data = []
with open( error_grid_file, "r" ) as fp:
for P in Config.order_values:
data.append([])
for grid in grids:
data[-1].append([])
for ewald in Config.ewald_values:
line = fp.readline()
err = float(line.strip().split()[-1])
data[-1][-1].append( err )
return data
def get_sampled_grid_sizes(npart, domain, box):
grids = get_grid_sizes(npart, domain, box)
test = Config.test
grids_at_interface = [ grids[p[2]] \
for p in get_interface_points( test.npart, test.domain, test.box ) ]
grids_at_interface = sorted(list(set(grids_at_interface)))
return grids_at_interface
def get_min_prediction_from_interface(interface_prediction, npart, domain,
box):
grids = get_grid_sizes(npart, domain, box)
# find minimum timing
min_est_t, min_est_params = min(interface_prediction)
min_est_ci, min_est_pi, min_est_gi, ewald_error = min_est_params
# extract parameters of fastest configuration
cutoff = Config.cutoff_values[min_est_ci]
P = Config.order_values[min_est_pi]
grid = grids[min_est_gi]
return (cutoff, P, grid, ewald_error[0][0], min_est_t)
def generate_kspace_error_config_file(test, output=sys.stdout):
print >> output, test.domain[0], test.domain[1], test.domain[2]
print >> output, test.npart
if test.accuracy:
print >> output, test.accuracy
else:
print >> output, test.accuracy_kspace
print >> output, Config.min_ewald, Config.max_ewald + Config.d_ewald / 10, Config.d_ewald
print >> output, Config.order_values[0], Config.order_values[-1]
grids = get_grid_sizes(test.npart, test.domain, test.box)
print >> output, len(grids)
for g in grids:
print >> output, "%4d %4d %4d" % g
def get_interface_points(npart, domain, box, width=1):
error_grid = get_accurate_grid(npart, domain, box, None)
grids = get_grid_sizes(npart, domain, box)
for cutoff_idx, cutoff in enumerate(Config.cutoff_values):
for P_idx, P in enumerate(Config.order_values):
for grid_idx, grid_side in enumerate(grids):
if error_grid[cutoff_idx][P_idx][grid_idx] and ( \
(cutoff_idx < width or not error_grid[cutoff_idx-width][P_idx][grid_idx]) and \
(P_idx < width or not error_grid[cutoff_idx][P_idx-width][grid_idx]) and \
(grid_idx < width or not error_grid[cutoff_idx][P_idx][grid_idx-width])
):
yield (cutoff_idx, P_idx, grid_idx,
error_grid[cutoff_idx][P_idx][grid_idx])
def get_prediction_with_statistics(npart, domain, box, mixing, np, nn, ts,
diff, acc):
estimated_timings = get_whole_interface_prediction(npart, domain, box,
mixing, np, nn, ts,
diff, acc)
empirical_timings = get_interface_timings(npart, domain, box, mixing, np,
nn, ts, diff, acc)
grids = get_grid_sizes(npart, domain, box)
differences = []
for i, (est, emp) in enumerate(zip(estimated_timings, empirical_timings)):
t_est = est[0][0]
#
cutoff = Config.cutoff_values[est[1][0]]
P = Config.order_values[est[1][1]]
grid = grids[est[1][2]]
print "%2d | %6.3f %6.3f %6.3f %6.3f | %4.2f %d (%2d, %2d %3d)" % (
i, t_est, est[0][1], est[0][3], est[0][2], cutoff, P, grid[0],
grid[1], grid[2])
t_emp = emp[0][0]
#t_emp = emp[0][1] + emp[0][2]
print " | %6.3f %6.3f %6.3f %6.3f | %4.2f %d (%2d, %2d %3d)" % (
t_emp, emp[0][1], emp[0][3], emp[0][2], cutoff, P, grid[0],
grid[1], grid[2])
#print " | %6.3f %6.3f %6.3f | %4.2f %d (%2d, %2d %3d)" % (emp[0][0], emp[0][1], emp[0][2], cutoff, P, grid[0], grid[1], grid[2] )
print
differences.append(abs(t_emp - t_est))
min_diff = min(differences)
max_diff = max(differences)
mean_diff = numpy.mean(differences)
median_diff = numpy.median(differences)
std_diff = numpy.std(differences)
min_estimated = get_min_prediction_from_interface(estimated_timings, npart,
domain, box)
return (min_estimated, (min_diff, max_diff, mean_diff, median_diff,
std_diff), (estimated_timings, empirical_timings))
def get_interface_timings(npart, domain, box, mixing, np, nn, ts, diff,
accuracy):
diff = diff
mixing = mixing
grids = get_grid_sizes(npart, domain, box)
timings = []
for (cutoff_idx, P_idx, grid_idx,
ewald_error) in get_interface_points(npart, domain, box):
# extract parameters
cutoff = Config.cutoff_values[cutoff_idx]
P = Config.order_values[P_idx]
grid = grids[grid_idx]
ewald = ewald_error[0][0]
# load data
output_dir = Config.get_test_timings_dir(Config.test)
fp = os.path.join(
output_dir, "Output-%s-%s-%.2f-%.2f-%d-%dx%dx%d" %
(diff, mixing, ewald, cutoff, P, grid[0], grid[1], grid[2]))
#t = extract_timings_from_log_full( fp ) # tuple (total, real, kspace)
t = extract_timings_from_log(fp) # tuple (total, real, kspace)
# store
timings.append((t, (cutoff_idx, P_idx, grid_idx, ewald_error)))
#print timings[-1]
return timings
def get_accurate_grid( N, domain, box, _ ):
c = [2.0 for i in range(N)]
C = sum([ch**2 for ch in c])
# Either accuracy is set, or the other two are
accuracy = Config.test.accuracy
acc_real = Config.test.accuracy_real
acc_kspace = Config.test.accuracy_kspace
#if not accuracy:
#acc_real = accuracy
if not acc_real:
acc_real = accuracy
if not acc_kspace:
acc_kspace = accuracy
error_real_grid = get_error_real_grid( box, N, C )
error_reciprocal_grid = get_error_reciprocal_grid( domain, box, N, C )
# Get min and max possible beta, given the ranges in Config
# and the desired accuracy
beta_idx = 0
error = error_real_grid[-1][beta_idx]
while error > acc_real and beta_idx < len(Config.ewald_values)-1: #(len(error_real_grid[3][-1])-1):
beta_idx += 1
error = error_real_grid[-1][beta_idx]
if error > acc_real: # increase max_beta and recursively call?
raise Exception # improve
min_beta_idx = beta_idx
min_beta = Config.ewald_values[0] + beta_idx * Config.d_ewald
max_beta_idx = -1
for P_idx in range(len(Config.order_values)):
beta_idx = len(Config.ewald_values)-1 #len(error_reciprocal_grid[0])-1
error = error_reciprocal_grid[P_idx][-1][beta_idx]
while error > acc_kspace and beta_idx > 0:
beta_idx -= 1
error = error_reciprocal_grid[P_idx][-1][beta_idx]
if not error > acc_kspace and beta_idx > max_beta_idx:
max_beta_idx = beta_idx
if max_beta_idx == -1:
raise Exception # improve
max_beta = Config.ewald_values[0] + max_beta_idx * Config.d_ewald
# Calculate the error estimate within the accurate region
# The rest is set to 0
grids = get_grid_sizes( N, domain, box )
error_grid = [
[
[[] for i in range(len(grids))] for j in range(len(Config.order_values))
] for k in range(len(Config.cutoff_values))
]
# Could iterate over regions with a relaxed accuracy constraint
# and then check actual accuracy before counting it in for min and max flops
for beta_idx in range(min_beta_idx, max_beta_idx+1):
for cutoff_idx, cutoff in enumerate(Config.cutoff_values):
error_real = error_real_grid[cutoff_idx][beta_idx]
if error_real > acc_real: # could run backwards and cut as soon as "inaccurate"
continue
for P_idx, P in enumerate(Config.order_values):
for grid_idx, grid in enumerate(grids):
error_reciprocal = error_reciprocal_grid[P_idx][grid_idx][beta_idx]
if accuracy: # single overall, then norm
error = math.sqrt(error_real**2 + error_reciprocal**2)
accurate = error < accuracy
else: # Otherwise, satisfy error is less than acc_ksapce
error = error_reciprocal
accurate = error < acc_kspace
if accurate:
error_grid[cutoff_idx][P_idx][grid_idx].append( (
Config.ewald_values[0] + Config.d_ewald * beta_idx,
error
) )
return error_grid
def run_interface_timings(npart, domain, box, mixing, np, nn, ts, diffmode,
accuracy):
grids = get_grid_sizes(npart, domain, box)
nprocs = np
#Config.create_dir( timings_dir )
timings_dir = Config.get_test_timings_dir(Config.test)
Config.create_dir(timings_dir)
# LSF Job submission
lsf_config = JobScript.LSF_Config()
lsf_config.group = "aices"
lsf_config.jobname = "PPPM-Timings"
#lsf_config.time = "%d:00" % (min(120, 2.5*np))
lsf_config.time = "2:00"
lsf_config.memory = "1000"
lsf_config.nthreads = np
lsf_config.parallelism_type = "openmpi"
lsf_config.arch_string = "Harpertown"
# Command
lsf_config.command = []
# Lammps Config
lmp_config = PPPMUtilities.LammpsPPPMConfig()
lmp_config.npart = npart
lmp_config.timesteps = ts
lmp_config.mixing = mixing
lmp_config.diff = diffmode
for (cutoff_idx, P_idx, grid_idx,
ewald_error) in get_interface_points(npart, domain, box):
lmp_config.ewald = ewald_error[0][0]
lmp_config.cutoff = Config.cutoff_values[cutoff_idx]
lmp_config.order = Config.order_values[P_idx]
lmp_config.grid = grids[grid_idx]
print lmp_config.ewald
print lmp_config.cutoff, "(", cutoff_idx, ")"
print lmp_config.order, "(", P_idx, ")"
print lmp_config.grid, "(", grid_idx, ")"
lmp_infile = os.path.join(
timings_dir, "Lammps-%s-%s-%.2f-%.2f-%d-%dx%dx%d" %
(lmp_config.diff, lmp_config.mixing, lmp_config.ewald,
lmp_config.cutoff, lmp_config.order, lmp_config.grid[0],
lmp_config.grid[1], lmp_config.grid[2]))
lmp_config.write_config_file(lmp_infile)
# LSF
outputfile = os.path.join(
timings_dir, "Output-%s-%s-%.2f-%.2f-%d-%dx%dx%d" %
(lmp_config.diff, lmp_config.mixing, lmp_config.ewald,
lmp_config.cutoff, lmp_config.order, lmp_config.grid[0],
lmp_config.grid[1], lmp_config.grid[2]))
lsf_config.command.append(
textwrap.dedent("""
export OMP_NUM_THREADS=1
mpirun -np %d $HOME/MD-libs/lammps-22Jan14/src/lmp_openmpi -l /dev/null < %s > %s
""" % (np, lmp_infile, outputfile)))
# Script
lsf_config.command = "\n".join(lsf_config.command)
jobscript = os.path.join(timings_dir, "Jobscript-Interface")
lsf_config.write_jobscript(jobscript)
print "Submitting job (Interface timing)... ",
subprocess.check_call("bsub < %s" % jobscript, shell=True)
print "Done"