def benchmark(mst_binary, input_graph, out, rev, trial_num, for_time):
rel_input_graph = ppinput(input_graph)
if not print_benchmark(rel_input_graph, out, rev, trial_num, for_time):
trial_num = -1 # cancel logging
# determine how to save the output
kill_out = False
if for_time or out != '/dev/null':
# disable /dev/null for performance experiments so we can get the weight for now ...
if out == '/dev/null':
out = random_tmp_filename(10, 'weight-for-time')
kill_out = True
save_cmd = '> ' + out
else:
# save just the first line of output so we can get the weight
out = random_tmp_filename(10, 'weight')
save_cmd = '| head -n 1 > ' + out
kill_out = True
# run mst (and time it)
time_file = random_tmp_filename(10, 'time')
cmd = '/usr/bin/time -f %%U -o %s %s %s %s' % (time_file, mst_binary,
input_graph, save_cmd)
ret = os.system(cmd)
if ret != 0:
print >> sys.stderr, "mst exited with error: " + cmd
quiet_remove(time_file)
return
try:
time_sec = extract_answer(time_file)
except CheckerError, e:
print >> sys.stderr, "failed to read time file: " + str(e)
return
def benchmark(mst_binary, input_graph, out, rev, trial_num, for_time):
rel_input_graph = ppinput(input_graph)
if not print_benchmark(rel_input_graph, out, rev, trial_num, for_time):
trial_num = -1 # cancel logging
# determine how to save the output
kill_out = False
if for_time or out != '/dev/null':
# disable /dev/null for performance experiments so we can get the weight for now ...
if out == '/dev/null':
out = random_tmp_filename(10, 'weight-for-time')
kill_out = True
save_cmd = '> ' + out
else:
# save just the first line of output so we can get the weight
out = random_tmp_filename(10, 'weight')
save_cmd = '| head -n 1 > ' + out
kill_out = True
# run mst (and time it)
time_file = random_tmp_filename(10, 'time')
cmd = '/usr/bin/time -f %%U -o %s %s %s %s' % (time_file, mst_binary, input_graph, save_cmd)
ret = os.system(cmd)
if ret != 0:
print >> sys.stderr, "mst exited with error: " + cmd
quiet_remove(time_file)
return
try:
time_sec = extract_answer(time_file)
except CheckerError, e:
print >> sys.stderr, "failed to read time file: " + str(e)
return
def make_latest(xaxis, alg, rev, index, num_verts):
"""Updates the symlink which points to the latest data file for an algorithm"""
o = '../' + get_output_dat_name(xaxis, alg, rev, index, num_verts)[len(DATA_PATH):]
linkname = DATA_PATH + 'latest/%s-%s-%s-latest' % (xaxis, alg, str(num_verts))
quiet_remove(linkname)
os.symlink(o, linkname)
return linkname
def main():
usage = """usage: %prog [options]
Tests the performance of the MST implementation. Alternatively, when -g is used
with a special argument (below), this determines the weight of the MST.
GEN_ARGS can be arbitrary arguments (to test performance) or one of the
following special arguments to generate a complete graph (for MST weight
computation only):
edge,NUM_VERTICES: random uniform edge weights [0, 1]
locN,NUM_VERTICES: randomly position vertices in N-dimensional space with axis ranges [0,1]"""
parser = OptionParser(usage)
parser.add_option(
"-c",
"--check",
action="store_true",
default=False,
help=
"whether to check output using check_output.py (only for the first run; exits if the check fails)"
)
parser.add_option(
"-C",
"--check-exit-0",
action="store_true",
default=False,
help=
"same as -c, but exit with code 0 even if the correctness check fails")
parser.add_option(
"-g",
"--generate-input",
metavar="GEN_ARGS",
help=
"generate (and use as input) a graph from generate_input.py GEN_ARGS (one for each run); -mqt will also be passed"
)
parser.add_option(
"-G",
"--generate-temp-input",
metavar="GEN_ARGS",
help="same as -g, but delete the graph after this script is done")
parser.add_option("-i",
"--input-file",
metavar="FILE",
help="FILE which describes the graph to use as input")
parser.add_option(
"-l",
"--inputs-list-file",
metavar="FILE",
help=
"specifies where to log correctness info (which inputs list log file) [default: inferred]"
)
parser.add_option("-n",
"--num-runs",
metavar="N",
type="int",
default=1,
help="number of runs to execute [default: %default]")
parser.add_option(
"-o",
"--output-file",
metavar="FILE",
help=
"where to save the output MST (stdout prints to stdout) [default: do not save output]"
)
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="do not print extraneous info to stdout")
parser.add_option(
"-r",
"--rev",
help=
"SHA1 of the git revision to build the mst binary from [default: use existing binary and do not log]"
)
parser.add_option("-x",
"--dont-log",
action="store_true",
default=False,
help="do not log the result")
parser.add_option(
"-t",
"--trial-num",
type="int",
default=-1,
help=
"run/trial identifier [default: do not log the trial, so ignore it]")
(options, args) = parser.parse_args()
if len(args) > 0:
parser.error("too many arguments: none expected")
# reconcile -g and -G
cleanup_generated_input = False
if options.generate_temp_input is not None:
if options.generate_input is not None:
parser.error('only one of -g or -G may be supplied')
else:
options.generate_input = options.generate_temp_input
cleanup_generated_input = True
# get the input file
is_test_perf = True
gen_input_args = None
if options.generate_input is not None and options.input_file is not None:
parser.error("-g and -i are mutually exclusive")
elif options.input_file is not None:
input_graph = options.input_file
elif options.generate_input is not None:
s = options.generate_input.split(',', 2)
gen_type = s[0]
if gen_type == "edge":
if len(s) != 2:
parser.error(
'-g edge,NUM_VERTICES form requires exactly these args')
gen_input_args = "-p 15 -e 0.0,1.0 %s" % s[1]
elif len(gen_type) == 4 and gen_type[:3] == "loc":
if len(s) != 2:
parser.error(
'-g %s,NUM_VERTICES form requires exactly these args' %
gen_type)
d = gen_type[3:]
gen_input_args = "-p 15 -v %s,0.0,1.0 %s" % (d, s[1])
# if it was not a special case, just pass the args straight through
if gen_input_args is None:
gen_input_args = options.generate_input
is_test_perf = not is_input_for_part2(gen_input_args.split())
else:
is_test_perf = False
input_graph = __generate_input_graph(gen_input_args,
cleanup_generated_input)
else:
parser.error(
"at least one of -g and -i must be used to specify the input graph"
)
if options.num_runs < 1:
parser.error("-n must be at least 1")
if options.trial_num < 0:
options.dont_log = True
# get the mst binary we want to test with
mst_binary = random_tmp_filename(10, 'mst')
__files_to_cleanup.append(mst_binary)
if options.rev is None or options.rev.lower() == 'current':
options.dont_log = True # no logging allowed on binaries which aren't checked in to the repo
options.trial_num = -1
options.rev = "" # tells the script to just use the current revision
cmd = 'copy_and_build_from_rev.sh %s %s %s' % (get_path_to_mst_binary(),
mst_binary, options.rev)
if options.quiet:
cmd += ' > /dev/null'
ret = os.system(get_path_to_tools_root() + cmd) # exclude-from-submit
# include-with-submit ret = 0
# include-with-submit mst_binary = './mst'
if ret != 0:
print 'error: unable to copy and build the mst binary'
__cleanup_and_exit(ret)
# handle check-exit-0
check_fail_exit_code = -1
if options.check_exit_0:
options.check = True
check_fail_exit_code = 0
# prepare the output file
if options.output_file:
out = options.output_file
out_is_temporary = False
else:
if options.check:
out = random_tmp_filename(10, 'out-for-checker')
__files_to_cleanup.append(out)
out_is_temporary = True
else:
if options.inputs_list_file:
print >> sys.stderr, 'warning: -l does nothing unless -c is also specified'
out = "/dev/null"
# do the first run (and check the output if requested)
test_mst(is_test_perf, mst_binary, input_graph, out, not options.dont_log,
options.rev, options.trial_num)
if options.check:
rev = None if options.rev is "" else options.rev
run = None if options.trial_num < 0 else options.trial_num
try:
ret = check(input_graph, out, 1, False, rev, run,
options.inputs_list_file)
except CheckerError, e:
ret = INCORRECT
print >> sys.stderr, str(e)
if out_is_temporary:
quiet_remove(out)
if ret != CORRECT:
__cleanup_and_exit(check_fail_exit_code
) # incorrectness already reported by check()
else:
print 'Correct - output checks out!'
def __cleanup_and_exit(code=0):
for fn in __files_to_cleanup:
quiet_remove(fn)
if __input_graph_to_cleanup is not None:
quiet_remove(__input_graph_to_cleanup)
sys.exit(code)
# run mst (and time it)
time_file = random_tmp_filename(10, 'time')
cmd = '/usr/bin/time -f %%U -o %s %s %s %s' % (time_file, mst_binary,
input_graph, save_cmd)
ret = os.system(cmd)
if ret != 0:
print >> sys.stderr, "mst exited with error: " + cmd
quiet_remove(time_file)
return
try:
time_sec = extract_answer(time_file)
except CheckerError, e:
print >> sys.stderr, "failed to read time file: " + str(e)
return
quiet_remove(time_file)
# try to get the weight (if we output the result somewhere)
mst_weight = -1.0
if out != "/dev/null":
try:
mst_weight = extract_answer(out)
except CheckerError, e:
print >> sys.stderr, "failed to read weight file: " + str(e)
return
str_mst_weight = ' mst_weight=' + str(mst_weight)
if kill_out:
quiet_remove(out) # was a temporary file we created
else:
str_mst_weight = ''
def gather_perf_data(alg, rev, index, latest):
"""Gathers performance data for a single revision of an algorithm"""
print 'gathering perf data for %s (rev=%s index=%u latest=%s)' % (alg, rev, index, str(latest))
# get the results
results = {} # maps (|V|, |E|) to ResultAccumulator
ds = DataSet.read_from_file(PerfResult, PerfResult.get_path_to(rev))
for data in ds.dataset.values():
key = (data.input().num_verts, data.input().num_edges)
result = results.get(key)
if result is None:
result = ResultAccumulator(data.time_sec)
result.defaultCI = DEFAULT_CI
results[key] = result
else:
result.add_data(data.time_sec)
# put the results in order
keys_density = results.keys()
keys_density.sort(density_compare)
keys_pom = results.keys()
keys_pom.sort(pom_compare)
keys = {}
keys['density'] = keys_density
keys['pom'] = keys_pom
# compute stats for all the results
for num_verts in results.keys():
results[num_verts].compute_stats()
# generate dat files for each x-axis cross important vertex counts
for xaxis in keys:
if xaxis == 'pom':
computex = lambda v, e : get_percent_of_max(v, e)
elif xaxis == 'density':
computex = lambda v, e : get_density(v, e)
else:
print >> sys.stderr, "unexpected x-axis value: " + str(xaxis)
sys.exit(-1)
header_txt = '#|V|\t|E|\t' + xaxis + '\tLower\tAverage\tUpper\t#Runs (Lower/Upper from ' + str(DEFAULT_CI) + '% CI)'
for vip in IMPORTANT_VERTS:
# open a file to output to
dat = get_output_dat_name(xaxis, alg, rev, index, vip)
print 'creating ' + dat
if latest:
latest_fn = make_latest(xaxis, alg, rev, index, vip)
try:
fh = open(dat, 'w')
# compute relevant stats and output them
print >> fh, header_txt
count = 0
for (v, e) in keys[xaxis]:
if vip=='all' or vip==v:
count += 1
r = results[(v, e)]
x = computex(v, e)
print >> fh, '%u\t%u\t%.6f\t%.3f\t%.3f\t%.3f\t%u' % (v, e, x, r.lower99, r.mean, r.upper99, len(r.values))
fh.close()
# don't create empty files
if count == 0:
quiet_remove(dat)
if latest:
quiet_remove(latest_fn)
except IOError, e:
print sys.stderr, "failed to write file: " + str(e)
return -1
def main():
usage = """usage: %prog [options]
Tests the performance of the MST implementation. Alternatively, when -g is used
with a special argument (below), this determines the weight of the MST.
GEN_ARGS can be arbitrary arguments (to test performance) or one of the
following special arguments to generate a complete graph (for MST weight
computation only):
edge,NUM_VERTICES: random uniform edge weights [0, 1]
locN,NUM_VERTICES: randomly position vertices in N-dimensional space with axis ranges [0,1]"""
parser = OptionParser(usage)
parser.add_option("-c", "--check",
action="store_true", default=False,
help="whether to check output using check_output.py (only for the first run; exits if the check fails)")
parser.add_option("-C", "--check-exit-0",
action="store_true", default=False,
help="same as -c, but exit with code 0 even if the correctness check fails")
parser.add_option("-g", "--generate-input",
metavar="GEN_ARGS",
help="generate (and use as input) a graph from generate_input.py GEN_ARGS (one for each run); -mqt will also be passed")
parser.add_option("-G", "--generate-temp-input",
metavar="GEN_ARGS",
help="same as -g, but delete the graph after this script is done")
parser.add_option("-i", "--input-file",
metavar="FILE",
help="FILE which describes the graph to use as input")
parser.add_option("-l", "--inputs-list-file",
metavar="FILE",
help="specifies where to log correctness info (which inputs list log file) [default: inferred]")
parser.add_option("-n", "--num-runs",
metavar="N", type="int", default=1,
help="number of runs to execute [default: %default]")
parser.add_option("-o", "--output-file",
metavar="FILE",
help="where to save the output MST (stdout prints to stdout) [default: do not save output]")
parser.add_option("-q", "--quiet",
action="store_true", default=False,
help="do not print extraneous info to stdout")
parser.add_option("-r", "--rev",
help="SHA1 of the git revision to build the mst binary from [default: use existing binary and do not log]")
parser.add_option("-x", "--dont-log",
action="store_true", default=False,
help="do not log the result")
parser.add_option("-t", "--trial-num",
type="int", default=-1,
help="run/trial identifier [default: do not log the trial, so ignore it]")
(options, args) = parser.parse_args()
if len(args) > 0:
parser.error("too many arguments: none expected")
# reconcile -g and -G
cleanup_generated_input = False
if options.generate_temp_input is not None:
if options.generate_input is not None:
parser.error('only one of -g or -G may be supplied')
else:
options.generate_input = options.generate_temp_input
cleanup_generated_input = True
# get the input file
is_test_perf = True
gen_input_args = None
if options.generate_input is not None and options.input_file is not None:
parser.error("-g and -i are mutually exclusive")
elif options.input_file is not None:
input_graph = options.input_file
elif options.generate_input is not None:
s = options.generate_input.split(',',2)
gen_type = s[0]
if gen_type == "edge":
if len(s) != 2:
parser.error('-g edge,NUM_VERTICES form requires exactly these args')
gen_input_args = "-p 15 -e 0.0,1.0 %s" % s[1]
elif len(gen_type)==4 and gen_type[:3] == "loc":
if len(s) != 2:
parser.error('-g %s,NUM_VERTICES form requires exactly these args' % gen_type)
d = gen_type[3:]
gen_input_args = "-p 15 -v %s,0.0,1.0 %s" % (d, s[1])
# if it was not a special case, just pass the args straight through
if gen_input_args is None:
gen_input_args = options.generate_input
is_test_perf = not is_input_for_part2(gen_input_args.split())
else:
is_test_perf = False
input_graph = __generate_input_graph(gen_input_args, cleanup_generated_input)
else:
parser.error("at least one of -g and -i must be used to specify the input graph")
if options.num_runs < 1:
parser.error("-n must be at least 1")
if options.trial_num < 0:
options.dont_log = True
# get the mst binary we want to test with
mst_binary = random_tmp_filename(10, 'mst')
__files_to_cleanup.append(mst_binary)
if options.rev is None or options.rev.lower() == 'current':
options.dont_log = True # no logging allowed on binaries which aren't checked in to the repo
options.trial_num = -1
options.rev = "" # tells the script to just use the current revision
cmd = 'copy_and_build_from_rev.sh %s %s %s' % (get_path_to_mst_binary(), mst_binary, options.rev)
if options.quiet:
cmd += ' > /dev/null'
ret = os.system(get_path_to_tools_root() + cmd) # exclude-from-submit
# include-with-submit ret = 0
# include-with-submit mst_binary = './mst'
if ret != 0:
print 'error: unable to copy and build the mst binary'
__cleanup_and_exit(ret)
# handle check-exit-0
check_fail_exit_code = -1
if options.check_exit_0:
options.check = True
check_fail_exit_code = 0
# prepare the output file
if options.output_file:
out = options.output_file
out_is_temporary = False
else:
if options.check:
out = random_tmp_filename(10, 'out-for-checker')
__files_to_cleanup.append(out)
out_is_temporary = True
else:
if options.inputs_list_file:
print >> sys.stderr, 'warning: -l does nothing unless -c is also specified'
out = "/dev/null"
# do the first run (and check the output if requested)
test_mst(is_test_perf, mst_binary, input_graph, out, not options.dont_log, options.rev, options.trial_num)
if options.check:
rev = None if options.rev is "" else options.rev
run = None if options.trial_num < 0 else options.trial_num
try:
ret = check(input_graph, out, 1, False, rev, run, options.inputs_list_file)
except CheckerError, e:
ret = INCORRECT
print >> sys.stderr, str(e)
if out_is_temporary:
quiet_remove(out)
if ret != CORRECT:
__cleanup_and_exit(check_fail_exit_code) # incorrectness already reported by check()
else:
print 'Correct - output checks out!'
def __cleanup_and_exit(code=0):
for fn in __files_to_cleanup:
quiet_remove(fn)
if __input_graph_to_cleanup is not None:
quiet_remove(__input_graph_to_cleanup)
sys.exit(code)
kill_out = True
# run mst (and time it)
time_file = random_tmp_filename(10, 'time')
cmd = '/usr/bin/time -f %%U -o %s %s %s %s' % (time_file, mst_binary, input_graph, save_cmd)
ret = os.system(cmd)
if ret != 0:
print >> sys.stderr, "mst exited with error: " + cmd
quiet_remove(time_file)
return
try:
time_sec = extract_answer(time_file)
except CheckerError, e:
print >> sys.stderr, "failed to read time file: " + str(e)
return
quiet_remove(time_file)
# try to get the weight (if we output the result somewhere)
mst_weight = -1.0
if out != "/dev/null":
try:
mst_weight = extract_answer(out)
except CheckerError, e:
print >> sys.stderr, "failed to read weight file: " + str(e)
return
str_mst_weight = ' mst_weight=' + str(mst_weight)
if kill_out:
quiet_remove(out) # was a temporary file we created
else:
str_mst_weight = ''
def main(argv=sys.argv):
if len(argv) != 2:
die("""usage: check_with_diff.py INPUT_GRAPH
Checks the current mst for a given input file and reports the diff, if any. It
does not do anything smart in terms of caching the correctness checker's output,
so it will be recomputed every time this is run.
""")
else:
input_graph = argv[1]
if not os.path.exists(input_graph):
die("%s not found" % input_graph)
# get our mst output
mst = get_path_to_mst_binary(make_sure_it_exists=False)
mst_out = random_tmp_filename(10)
if os.system('%s %s > %s' % (mst, input_graph, mst_out)) != 0:
quiet_remove(mst_out)
die("failed to run mst (or exited with an error code)")
# get the checker's output
checker = get_path_to_checker_binary(make_sure_it_exists=True)
checker_out = random_tmp_filename(10)
if os.system('%s %s true > %s' % (checker, input_graph, checker_out)) != 0:
quiet_remove(mst_out)
quiet_remove(checker_out)
die("failed to run checker (or exited with an error code)")
# check just the MST weight first
mst_w = extract_answer(mst_out)
checker_w = extract_answer(checker_out)
fmt = '%.1f' # tolerance to 1 decimal place
str_mst_w = fmt % mst_w
str_checker_w = fmt % checker_w
if str_mst_w == str_checker_w:
print 'Weights match! %s %s' % (str_mst_w, str_checker_w)
quiet_remove(mst_out)
quiet_remove(checker_out)
return 0
else:
print 'Weight mistmatch, comparing vertices!! (checker_mst (%s) - our_mst (%s) = %s)' % (
str(checker_w), str(mst_w), str(checker_w - mst_w))
# sort them
mst_out2 = random_tmp_filename(10)
checker_out2 = random_tmp_filename(10)
sort_and_order = get_path_to_project_root(
) + 'src/input/sort_and_order.py '
os.system(sort_and_order + mst_out + ' 1 > ' + mst_out2)
os.system(sort_and_order + checker_out + ' 1 > ' + checker_out2)
# compare them
os.system('diff %s %s && echo Edges are the same!' %
(checker_out2, mst_out2))
quiet_remove(mst_out)
quiet_remove(mst_out2)
quiet_remove(checker_out)
quiet_remove(checker_out2)
return 0
def main(argv=sys.argv):
if len(argv) != 2:
die("""usage: check_with_diff.py INPUT_GRAPH
Checks the current mst for a given input file and reports the diff, if any. It
does not do anything smart in terms of caching the correctness checker's output,
so it will be recomputed every time this is run.
""")
else:
input_graph = argv[1]
if not os.path.exists(input_graph):
die("%s not found" % input_graph)
# get our mst output
mst = get_path_to_mst_binary(make_sure_it_exists=False)
mst_out = random_tmp_filename(10)
if os.system('%s %s > %s' % (mst, input_graph, mst_out)) != 0:
quiet_remove(mst_out)
die("failed to run mst (or exited with an error code)")
# get the checker's output
checker = get_path_to_checker_binary(make_sure_it_exists=True)
checker_out = random_tmp_filename(10)
if os.system('%s %s true > %s' % (checker, input_graph, checker_out)) != 0:
quiet_remove(mst_out)
quiet_remove(checker_out)
die("failed to run checker (or exited with an error code)")
# check just the MST weight first
mst_w = extract_answer(mst_out)
checker_w = extract_answer(checker_out)
fmt = '%.1f' # tolerance to 1 decimal place
str_mst_w = fmt % mst_w
str_checker_w = fmt % checker_w
if str_mst_w == str_checker_w:
print 'Weights match! %s %s' % (str_mst_w, str_checker_w)
quiet_remove(mst_out)
quiet_remove(checker_out)
return 0
else:
print 'Weight mistmatch, comparing vertices!! (checker_mst (%s) - our_mst (%s) = %s)' % (str(checker_w), str(mst_w), str(checker_w - mst_w))
# sort them
mst_out2 = random_tmp_filename(10)
checker_out2 = random_tmp_filename(10)
sort_and_order = get_path_to_project_root() + 'src/input/sort_and_order.py '
os.system(sort_and_order + mst_out + ' 1 > ' + mst_out2)
os.system(sort_and_order + checker_out + ' 1 > ' + checker_out2)
# compare them
os.system('diff %s %s && echo Edges are the same!' % (checker_out2, mst_out2))
quiet_remove(mst_out)
quiet_remove(mst_out2)
quiet_remove(checker_out)
quiet_remove(checker_out2)
return 0