def test_named_timer(profiling_env):
buf = StringIO()
prof.start("foo")
prof.stop("foo", file=buf)
out = buf.getvalue()
assert " foo " == re.search(" foo ", out).group(0)
def test_named_timer(self):
buf = StringIO()
prof.start("foo")
prof.stop("foo", file=buf)
out = buf.getvalue()
self._test(" foo ", re.search(" foo ", out).group(0))
def eval_classifier_instance(job, db, classifier, instance,
err_fn, training):
# Get relevant values from instance.
oracle = instance.get_string_value(instance.class_index)
scenario = instance.get_string_value(0)
# Get default value.
try:
baseline = training.default
except AttributeError:
training.default = get_one_r(db, training)
baseline = training.default
# Classify instance, and convert to params ID.
prof.start()
value = classifier.classify_instance(instance)
elapsed = prof.elapsed()
attr = instance.dataset.attribute(instance.class_index)
predicted = attr.value(value)
correct = 1 if predicted == oracle else 0
if correct:
illegal = 0
refused = 0
performance, speedup = perf_fn(db, scenario, predicted,
oracle, baseline)
else:
# Determine if predicted workgroup size is valid or not. A
# valid predicted is one which is within the max_wgsize for
# that particular instance.
max_wgsize_attr = instance.dataset.attribute_by_name("kern_max_wg_size")
max_wgsize_attr_index = max_wgsize_attr.index
max_wgsize = int(instance.get_value(max_wgsize_attr_index))
wg_c, wg_r = unhash_params(predicted)
illegal = 0 if wg_c * wg_r < max_wgsize else 1
if illegal:
refused = 0
else:
try:
db.runtime(scenario, predicted)
refused = 0
except lab.db.Error:
refused = 1
if illegal or refused:
predicted = err_fn(instance, max_wgsize, wg_c, wg_r, baseline)
performance, speedup = perf_fn(db, scenario, predicted,
oracle, baseline)
db.add_classification_result(job, classifier,
err_fn, training, scenario,
oracle, predicted, baseline, correct, illegal,
refused, performance, speedup, elapsed)
def get_oclverified(self, s: session_t) -> bool:
if self.oclverified == None:
prof.start("dsmith oclgrind")
testcase = s.query(Testcase) \
.filter(Testcase.id == self.id) \
.scalar()
self.oclverified = oclgrind.verify_dsmith_testcase(testcase)
s.commit()
prof.stop("dsmith oclgrind")
return self.oclverified
def eval_classifiers(db, classifiers, err_fns, job, training, testing):
"""
Cross validate a set of classifiers and err_fns.
"""
for classifier in classifiers:
meta = Classifier(classifier)
prof.start("train classifier")
meta.build_classifier(training)
prof.stop("train classifier")
basename = ml.classifier_basename(classifier.classname)
for err_fn in err_fns:
io.debug(job, basename, err_fn.func.__name__, testing.num_instances)
for j,instance in enumerate(testing):
eval_classifier_instance(job, db, meta, instance, err_fn,
training)
db.commit()
def get_gpuverified(self, s: session_t) -> bool:
if self.gpuverified == None:
prof.start("dsmith gpuverify")
src = s.query(Program.src) \
.join(Testcase) \
.filter(Testcase.id == self.id) \
.scalar()
try:
clgen.gpuverify(src, ["--local_size=64", "--num_groups=128"])
self.gpuverified = True
except clgen.GPUVerifyException:
self.gpuverified = False
s.commit()
prof.stop("dsmith gpuverify")
return self.gpuverified
def eval_linear_models(db, models):
rows = db.execute(
"SELECT "
" scenario_stats.scenario, "
" kernels.max_wg_size, "
" scenario_stats.oracle_param "
"FROM scenarios "
"LEFT JOIN scenario_stats "
" ON scenarios.id=scenario_stats.scenario "
"LEFT JOIN kernels "
" ON scenarios.kernel=kernels.id"
).fetchall()
baseline = db.one_r()[0]
prof.start("Linear models")
for scenario,max_wgsize,oracle in rows:
for model in models:
wg_c, wg_r = model.predict(scenario, max_wgsize, oracle)
try:
prediction = hash_params(wg_c, wg_r)
illegal = 0 if wg_c * wg_r < max_wgsize else 1
correct = 1 if prediction == oracle else 0
db.runtime(scenario, prediction)
refused = 0
reshape_param = prediction
reshape_perf, reshape_speedup = perf_fn(db, scenario,
prediction, oracle,
baseline)
baseline_perf, baseline_speedup = reshape_perf, reshape_speedup
random_param = prediction
random_perf, random_speedup = reshape_perf, reshape_speedup
except lab.db.Error:
refused = not illegal
reshape_param = reshape(db, scenario, max_wgsize, wg_c, wg_r)
reshape_perf, reshape_speedup = perf_fn(db, scenario,
reshape_param,
oracle, baseline)
baseline_perf, baseline_speedup = perf_fn(db, scenario,
baseline, oracle,
baseline)
random_param = random.choice(db.W_legal(scenario))
random_perf, random_speedup = perf_fn(db, scenario,
random_param, oracle,
baseline)
db.add_model_result(model.id(), "reshape_fn", scenario, oracle,
reshape_param, correct, illegal, refused,
reshape_perf, reshape_speedup)
db.add_model_result(model.id(), "default_fn", scenario, oracle,
baseline, correct, illegal, refused,
baseline_perf, baseline_speedup)
db.add_model_result(model.id(), "random_fn", scenario, oracle,
random_param, correct, illegal, refused,
random_perf, random_speedup)
db.commit()
prof.stop("Linear models")
def eval_speedup_regressors(db, classifiers, baseline, rank_fn,
table, job, training, testing):
maxwgsize_index = testing.attribute_by_name("kern_max_wg_size").index
wg_c_index = testing.attribute_by_name("wg_c").index
wg_r_index = testing.attribute_by_name("wg_r").index
insert_str = ("INSERT INTO {} VALUES "
"(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)".format(table))
for classifier in classifiers:
meta = Classifier(classifier)
prof.start("train classifier")
meta.build_classifier(training)
prof.stop("train classifier")
basename = ml.classifier_basename(classifier.classname)
classifier_id = db.classifier_id(classifier)
io.debug(job, basename, testing.num_instances)
scenarios = set([instance.get_string_value(0)
for instance in testing])
instances = zip(scenarios, [
(instance for instance in testing if
instance.get_string_value(0) == scenario).next()
for scenario in scenarios
])
for scenario,instance in instances:
maxwgsize = int(instance.get_value(maxwgsize_index))
wlegal = space.enumerate_wlegal_params(maxwgsize)
predictions = []
elapsed = 0
for params in wlegal:
wg_c, wg_r = unhash_params(params)
instance.set_value(wg_c_index, wg_c)
instance.set_value(wg_r_index, wg_r)
# Predict the speedup for a particular set of
# parameters.
prof.start()
predicted = meta.classify_instance(instance)
elapsed += prof.elapsed()
predictions.append((params, predicted))
# Rank the predictions from highest to lowest speedup.
predictions = sorted(predictions, key=lambda x: x[1], reverse=True)
row = db.execute(
"SELECT "
" oracle_param,"
" ("
" SELECT mean FROM runtime_stats "
" WHERE scenario=? AND params=?"
" ) * 1.0 / oracle_runtime AS oracle_speedup,"
" worst_runtime / oracle_runtime AS actual_range "
"FROM scenario_stats "
"WHERE scenario=?",
(scenario,baseline,scenario)).fetchone()
actual = row[:2]
predicted_range = predictions[-1][1] - predictions[0][1]
actual_range = row[2] - row[1]
num_attempts = 1
while True:
predicted = predictions.pop(0)
try:
speedup = db.speedup(scenario, baseline, predicted[0])
perf = db.perf(scenario, predicted[0])
try:
speedup_he = self.speedup(scenario, HE_PARAM, predicted[0])
except:
speedup_he = None
try:
speedup_mo = self.speedup(scenario, MO_PARAM, predicted[0])
except:
speedup_mo = None
db.execute(insert_str,
(job, classifier_id, scenario, actual[0],
actual[1], predicted[0], predicted[1],
actual_range, predicted_range,
num_attempts,
1 if predicted[0] == actual[0] else 0,
perf, speedup, speedup_he, speedup_mo, elapsed))
break
except _db.MissingDataError:
num_attempts += 1
pass
db.commit()
def test_timers(profiling_env):
x = len(list(prof.timers()))
prof.start("new timer")
assert len(list(prof.timers())) == x + 1
prof.stop("new timer")
assert len(list(prof.timers())) == x
def test_stop_twice_error(profiling_env):
prof.start("foo")
prof.stop("foo")
with pytest.raises(KeyError):
prof.stop("foo")
def test_timers(self):
x = len(list(prof.timers()))
prof.start("new timer")
self.assertEqual(len(list(prof.timers())), x + 1)
prof.stop("new timer")
self.assertEqual(len(list(prof.timers())), x)
def test_stop_twice_error(self):
prof.start("foo")
prof.stop("foo")
with self.assertRaises(KeyError):
prof.stop("foo")
