def run_session(self):
params_set = dict(self.planner_config[self.planner].items())
manipulator = self._load_search_space(params_set)
interface = DefaultMeasurementInterface(args=self.args,
manipulator=manipulator)
# Using OpenTuner API
# https://github.com/jansel/opentuner/blob/master/examples/py_api/api_example.py
api = TuningRunManager(interface, self.args)
self.n_trial = 0 # Reset to n_trials to zero for each planner
start_time = timer()
params = {'planner': self.planner, 'start_time': start_time}
if (self.MAX_RUNTIME != 'None'):
params['end_time'] = timer() + self.MAX_RUNTIME
print('\n')
rospy.loginfo('Executing %s on %s for %d secs', self.MODE,
params['planner'], self.MAX_RUNTIME)
else:
print('\n')
rospy.loginfo('Executing %s on %s for %d trials', self.MODE,
params['planner'], self.MAX_TRIALS)
for _ in range(self.MAX_TRIALS):
desired_result = api.get_next_desired_result()
params_set = desired_result.configuration.data
params['params_set'] = params_set
run_result = self._opentuner_obj(params)
result = Result(time=run_result['loss'])
api.report_result(desired_result, result)
def __init__(self, *ot_args, **ot_kwargs):
"""
Creates communication queues and spawn a thread
to run the tuning logic.
"""
super(OpenTunerDriver, self).__init__()
self._best_config = None
interface = CtreeMeasurementInterface(self, *ot_args, **ot_kwargs)
arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers())
config_args = CONFIG.get("opentuner", "args").split()
tuner_args = arg_parser.parse_args(config_args)
self.manager = TuningRunManager(interface, tuner_args)
self._converged = False
def main():
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('x', -200, 200))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='examples',
program_name='api_test',
program_version='0.1')
api = TuningRunManager(interface, args)
for x in xrange(500):
desired_result = api.get_next_desired_result()
if desired_result is None:
# The search space for this example is very small, so sometimes
# the techniques have trouble finding a config that hasn't already
# been tested. Change this to a continue to make it try again.
break
cfg = desired_result.configuration.data
result = Result(time=test_func(cfg))
api.report_result(desired_result, result)
best_cfg = api.get_best_configuration()
api.finish()
print 'best x found was', best_cfg['x']
def __init__(self, train_queries, test_queries, experiment_config,
result_dir, system_environment):
"""
Args:
train_queries (list of queries (SQLQuery queries))
test_queries (list of queries (SQLQuery queries))
result_dir (str)
system (PostgresSystemEnvironment): encapsulates environment
"""
self.logger = logging.getLogger(__name__)
self.logger.info('Setting up OpenTuner...')
self.train_queries = train_queries
self.test_queries = test_queries
self.experiment_config = experiment_config
self.max_size = experiment_config['max_size']
self.size_weight = experiment_config['size_weight']
self.max_runtime = experiment_config['max_runtime']
self.runtime_weight = experiment_config['runtime_weight']
self.reward_penalty = experiment_config['reward_penalty']
self.system_environment = system_environment
self.result_dir = result_dir
# 2nd search space
self.n_idxs = experiment_config['n_queries_per_episode']
self.n_cols_per_idx = 3 # TODO hardcode
self.tbls = experiment_config['tables']
self.n_idxs_per_tbl = int(self.n_idxs / len(self.tbls))
# maps tbls to mapping of tbl's column indices to tbl's columns, per search space representation
self.tbl_2_col_idx_2_col = {}
for tbl in self.tbls:
self.tbl_2_col_idx_2_col[tbl] = {}
for col_idx, col in enumerate(tpch_table_columns[tbl].keys()):
self.tbl_2_col_idx_2_col[tbl][col_idx +
1] = col # + 1 b/c 0 is noop
# api
sys.argv = [sys.argv[0]] # opentuner expects own args
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
manipulator = self.build_search_space()
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator)
self.api = TuningRunManager(interface, args)
def main():
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('x', -200, 200))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='examples',
program_name='api_test',
program_version='0.1')
api = TuningRunManager(interface, args)
for x in range(500):
desired_result = api.get_next_desired_result()
if desired_result is None:
# The search space for this example is very small, so sometimes
# the techniques have trouble finding a config that hasn't already
# been tested. Change this to a continue to make it try again.
break
cfg = desired_result.configuration.data
result = Result(time=test_func(cfg))
api.report_result(desired_result, result)
best_cfg = api.get_best_configuration()
api.finish()
print('best x found was', best_cfg['x'])
def create_benchmark_api(benchmark):
args = argparser.parse_args()
args.benchmark = benchmark
args.database = "{}_{}{}".format(
os.path.splitext(args.database)[0], benchmark,
os.path.splitext(args.database)[1])
interface = ReproBLASTuner(args)
api = TuningRunManager(interface, args)
return api
class OpenTunerDriver(TuningDriver):
"""
Object that interacts with backend tuners. Provides
a stream of configurations, as well as an interface
to report on the performance of each.
"""
def __init__(self, *ot_args, **ot_kwargs):
"""
Creates communication queues and spawn a thread
to run the tuning logic.
"""
super(OpenTunerDriver, self).__init__()
self._best_config = None
interface = CtreeMeasurementInterface(self, *ot_args, **ot_kwargs)
arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers())
config_args = CONFIG.get("opentuner", "args").split()
tuner_args = arg_parser.parse_args(config_args)
self.manager = TuningRunManager(interface, tuner_args)
self._converged = False
def _get_configs(self):
"""Get the next configuration to test."""
timeout = CONFIG.getint("opentuner", "timeout")
while True:
self.curr_desired_result = self.manager.get_next_desired_result()
if self.curr_desired_result is None:
break
yield self.curr_desired_result.configuration.data
print("Best configuration", self.manager.get_best_configuration())
log.info("exhausted stream of configurations.")
best_config = self.manager.get_best_configuration()
assert best_config != None, "No best configuration reported."
self._converged = True
while True:
yield best_config
def report(self, **kwargs):
"""Report the performance of the most recent configuration."""
if not self._converged:
print("Tuning run result:",
self.curr_desired_result.configuration.data, kwargs)
self.manager.report_result(self.curr_desired_result,
Result(**kwargs))
class OpenTunerDriver(TuningDriver):
"""
Object that interacts with backend tuners. Provides
a stream of configurations, as well as an interface
to report on the performance of each.
"""
def __init__(self, *ot_args, **ot_kwargs):
"""
Creates communication queues and spawn a thread
to run the tuning logic.
"""
super(OpenTunerDriver, self).__init__()
self._best_config = None
interface = CtreeMeasurementInterface(self, *ot_args, **ot_kwargs)
arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers())
config_args = CONFIG.get("opentuner", "args").split()
tuner_args = arg_parser.parse_args(config_args)
self.manager = TuningRunManager(interface, tuner_args)
self._converged = False
def _get_configs(self):
"""Get the next configuration to test."""
timeout = CONFIG.getint("opentuner", "timeout")
while True:
self.curr_desired_result = self.manager.get_next_desired_result()
if self.curr_desired_result is None:
break
yield self.curr_desired_result.configuration.data
print("Best configuration", self.manager.get_best_configuration())
log.info("exhausted stream of configurations.")
best_config = self.manager.get_best_configuration()
assert best_config != None, "No best configuration reported."
self._converged = True
while True:
yield best_config
def report(self, **kwargs):
"""Report the performance of the most recent configuration."""
if not self._converged:
print("Tuning run result:", self.curr_desired_result.configuration.data, kwargs)
self.manager.report_result(self.curr_desired_result, Result(**kwargs))
def create_test_tuning_run(db):
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
args.database = db
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('x', -200, 200))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='examples',
program_name='api_test',
program_version='0.1')
api = TuningRunManager(interface, args)
return api
def __init__(self, *ot_args, **ot_kwargs):
"""
Creates communication queues and spawn a thread
to run the tuning logic.
"""
super(OpenTunerDriver, self).__init__()
self._best_config = None
interface = CtreeMeasurementInterface(self, *ot_args, **ot_kwargs)
arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers())
config_args = CONFIG.get("opentuner", "args").split()
tuner_args = arg_parser.parse_args(config_args)
self.manager = TuningRunManager(interface, tuner_args)
self._converged = False
import argparse
def get_next_desired_result():
global desired_result
desired_result = api.get_next_desired_result()
while desired_result is None:
desired_result = api.get_next_desired_result()
return desired_result.configuration.data
def report_result(runtime):
api.report_result(desired_result, Result(time=runtime))
def finish():
api.finish()
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
manipulator = ConfigurationManipulator()
:::parameters:::
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='atf_library',
program_name='atf_library',
program_version='0.1')
api = TuningRunManager(interface, args)
)"
def tuning_loop():
report_delay = 30
last_time = time.time()
start_time = last_time
iterations = 5
parser = argparse.ArgumentParser(parents = opentuner.argparsers())
parser.add_argument("--processes",
type = int,
help = "Number of Python threads available.")
parser.add_argument("--no-wait",
action = "store_true",
help = "Do not wait for requested results to generate more requests.")
parser.add_argument("--application",
type = str,
help = "Application name.")
parser.add_argument("--verilog-file",
type = str,
help = "Verilog file for the application.")
args = parser.parse_args()
pool = ThreadPool(args.processes)
manipulator = ConfigurationManipulator()
global application
global verilog_file
global application_path
global container_path
global host_path
global image_name
global script_name
global tuning_init
application = args.application
verilog_file = args.verilog_file
application_path = "/root/legup_src/legup-4.0/examples/chstone/{0}".format(application)
container_path = "/root/legup_src/legup-4.0/examples/chstone/{0}/tuner".format(application)
host_path = "/home/bruelp/legup-tuner/post_place_and_route/py"
image_name = "legup_quartus"
script_name = "measure.sh"
print(application, container_path, application_path)
for name in legup_parameters.parameters:
parameter_type = legup_parameters.parameter_type(name)
values = legup_parameters.parameter_values(name)
if parameter_type == int:
manipulator.add_parameter(IntegerParameter(name, values[0], values[1]))
elif parameter_type == bool:
manipulator.add_parameter(BooleanParameter(name))
elif parameter_type == Enum:
manipulator.add_parameter(EnumParameter(name, values))
else:
print("ERROR: No such parameter type \"{0}\"".format(name))
interface = DefaultMeasurementInterface(args = args,
manipulator = manipulator,
project_name = 'HLS-FPGAs',
program_name = 'legup-tuner',
program_version = '0.0.1')
manager = TuningRunManager(interface, args)
current_time = time.time()
computing_results = []
computed_results = []
desired_results = manager.get_desired_results()
while current_time - start_time < args.stop_after:
if args.no_wait:
if len(desired_results) != 0 or len(computing_results) != 0:
for desired_result in desired_results:
computing_results.append([desired_result,
pool.apply_async(get_wallclock_time,
(desired_result.configuration.data, ))])
for result in computing_results:
if result[1].ready() and result[0] not in computed_results:
cost = result[1].get()
manager.report_result(result[0], Result(time = cost))
computed_results.append(result)
for result in computed_results:
if result in computing_results:
computing_results.remove(result)
computed_results = []
else:
if len(desired_results) != 0:
cfgs = [dr.configuration.data for dr in desired_results]
results = pool.map_async(get_wallclock_time, cfgs).get(timeout = None)
for dr, result in zip(desired_results, results):
manager.report_result(dr,
Result(time = result['value'],
cycles = result['cycles'],
fmax = result['fmax'],
LU = result['lu'],
pins = result['pins'],
regs = result['regs'],
block = result['block'],
ram = result['ram'],
dsp = result['dsp']))
desired_results = manager.get_desired_results()
current_time = time.time()
if (current_time - last_time) >= report_delay:
log_intermediate(current_time - start_time, manager)
last_time = current_time
current_time = time.time()
log_intermediate(current_time - start_time, manager)
save_final_configuration(manager.get_best_configuration())
manager.finish()
class OpentunerOptimizer(AbstractOptimizer):
primary_import = "opentuner"
def __init__(self,
api_config,
techniques=DEFAULT_TECHNIQUES,
n_suggestions=1):
"""Build wrapper class to use opentuner optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
techniques : iterable of strings
A list or tuple of techniques to use in opentuner. If the list
has only one technique, then that technique will be used. If the
list has multiple techniques a bandit over those techniques
will be used.
n_suggestions : int
Default number of suggestions to be made in parallel.
"""
AbstractOptimizer.__init__(self, api_config)
# Opentuner requires DesiredResult to reference suggestion when making
# its observation. x_to_dr maps the dict suggestion to DesiredResult.
self.x_to_dr = {}
# Keep last suggested x and repeat it whenever opentuner gives up.
self.dummy_suggest = None
"""Setting up the arguments for opentuner. You can see all possible
arguments using:
```
>>> import opentuner
>>> opentuner.default_argparser().parse_args(['-h'])
```
We only change a few arguments (other arguments are set to defaults):
* database = MEMORY_ONLY_DB: to use an in-memory sqlite database
* parallelism = n_suggestions: num of suggestions to give in parallel
* technique = techniques: a list of techniques to be used by opentuner
* print_params = False: to avoid opentuner from exiting after printing
param spaces
"""
args = Namespace(
bail_threshold=500,
database=MEMORY_ONLY_DB,
display_frequency=10,
generate_bandit_technique=False,
label=None,
list_techniques=False,
machine_class=None,
no_dups=False,
parallel_compile=False,
parallelism=n_suggestions,
pipelining=0,
print_params=False,
print_search_space_size=False,
quiet=False,
results_log=None,
results_log_details=None,
seed_configuration=[],
stop_after=None,
technique=techniques,
test_limit=5000,
)
# Setup some dummy classes required by opentuner to actually run.
manipulator = OpentunerOptimizer.build_manipulator(api_config)
interface = DMI(args=args, manipulator=manipulator)
self.api = TuningRunManager(interface, args)
@staticmethod
def hashable_dict(d):
"""A custom function for hashing dictionaries.
Parameters
----------
d : dict or dict-like
The dictionary to be converted to immutable/hashable type.
Returns
-------
hashable_object : frozenset of tuple pairs
Bijective equivalent to dict that can be hashed.
"""
hashable_object = frozenset(d.items())
return hashable_object
@staticmethod
def build_manipulator(api_config):
"""Build a ConfigurationManipulator object to be used by opentuner.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
Returns
-------
manipulator : ConfigurationManipulator
Some over complexified class required by opentuner to run.
"""
manipulator = ConfigurationManipulator()
for pname in api_config:
ptype = api_config[pname]["type"]
pspace = api_config[pname].get("space", None)
pmin, pmax = api_config[pname].get("range", (None, None))
if ptype == "real":
if pspace in ("linear", "logit"):
ot_param = FloatParameter(pname, pmin, pmax)
elif pspace in ("log", "bilog"):
LogFloatParameter_ = ClippedParam(LogFloatParameter)
ot_param = LogFloatParameter_(pname, pmin, pmax)
else:
assert False, "unsupported param space = %s" % pspace
elif ptype == "int":
if pspace in ("linear", "logit"):
ot_param = IntegerParameter(pname, pmin, pmax)
elif pspace in ("log", "bilog"):
ot_param = LogIntegerParameter(pname, pmin, pmax)
else:
assert False, "unsupported param space = %s" % pspace
elif ptype == "bool":
# The actual bool parameter seems not to work in Py3 :(
ot_param = IntegerParameter(pname, 0, 1)
elif ptype in ("cat", "ordinal"):
# Treat ordinal and categorical variables the same for now.
assert "values" in api_config[pname]
pvalues = api_config[pname]["values"]
ot_param = EnumParameter(pname, pvalues)
else:
assert False, "type=%s/space=%s not handled in opentuner yet" % (
ptype, pspace)
manipulator.add_parameter(ot_param)
return manipulator
def suggest(self, n_suggestions=1):
"""Make `n_suggestions` suggestions for what to evaluate next.
This requires the user observe all previous suggestions before calling
again.
Parameters
----------
n_suggestions : int
The number of suggestions to return.
Returns
-------
next_guess : list of dict
List of `n_suggestions` suggestions to evaluate the objective
function. Each suggestion is a dictionary where each key
corresponds to a parameter being optimized.
"""
assert n_suggestions >= 1, "invalid value for n_suggestions"
# Update the n_suggestions if it is different from the current setting.
if self.api.search_driver.args.parallelism != n_suggestions:
self.api.search_driver.args.parallelism = n_suggestions
warnings.warn("n_suggestions changed across suggest calls")
# Require the user to already observe all previous suggestions.
# Otherwise, opentuner will just recycle old suggestions.
assert len(
self.x_to_dr
) == 0, "all the previous suggestions should have been observed by now"
# The real meat of suggest from opentuner: Get next `n_suggestions`
# unique suggestions.
desired_results = [
self.api.get_next_desired_result() for _ in range(n_suggestions)
]
# Save DesiredResult object in dict since observe will need it.
X = []
using_dummy_suggest = False
for ii in range(n_suggestions):
# Opentuner can give up, but the API requires guessing forever.
if desired_results[ii] is None:
assert self.dummy_suggest is not None, "opentuner gave up on the first call!"
# Use the dummy suggestion in this case.
X.append(self.dummy_suggest)
using_dummy_suggest = True
continue
# Get the simple dict equivalent to suggestion.
x_guess = desired_results[ii].configuration.data
X.append(x_guess)
# Now save the desired result for future use in observe.
x_guess_ = OpentunerOptimizer.hashable_dict(x_guess)
assert x_guess_ not in self.x_to_dr, "the suggestions should not already be in the x_to_dr dict"
self.x_to_dr[x_guess_] = desired_results[ii]
# This will also catch None from opentuner.
assert isinstance(self.x_to_dr[x_guess_], DesiredResult)
assert len(
X
) == n_suggestions, "incorrect number of suggestions provided by opentuner"
# Log suggestion for repeating if opentuner gives up next time. We can
# only do this when it is not already being used since it we will be
# checking guesses against dummy_suggest in observe.
if not using_dummy_suggest:
self.dummy_suggest = X[-1]
return X
def observe(self, X, y):
"""Feed the observations back to opentuner.
Parameters
----------
X : list of dict-like
Places where the objective function has already been evaluated.
Each suggestion is a dictionary where each key corresponds to a
parameter being optimized.
y : array-like, shape (n,)
Corresponding values where objective has been evaluated.
"""
assert len(X) == len(y)
for x_guess, y_ in zip(X, y):
x_guess_ = OpentunerOptimizer.hashable_dict(x_guess)
# If we can't find the dr object then it must be the dummy guess.
if x_guess_ not in self.x_to_dr:
assert x_guess == self.dummy_suggest, "Appears to be guess that did not originate from suggest"
continue
# Get the corresponding DesiredResult object.
dr = self.x_to_dr.pop(x_guess_, None)
# This will also catch None from opentuner.
assert isinstance(
dr,
DesiredResult), "DesiredResult object not available in x_to_dr"
# Opentuner's arg names assume we are minimizing execution time.
# So, if we want to minimize we have to pretend y is a 'time'.
result = Result(time=y_)
self.api.report_result(dr, result)
def solve(self, job):
logging.debug('Starting opentuner')
failed_jobs_threshold = self.jobs_limit * _FAILED_JOBS_COEF
manipulator = ConfigurationManipulator()
for var in job.optimization_job.task_variables:
if var.HasField('continuous_variable'):
cont_var = var.continuous_variable
param = FloatParameter(var.name, cont_var.l, cont_var.r)
else:
int_var = var.integer_variable
param = IntegerParameter(var.name, int_var.l, int_var.r)
manipulator.add_parameter(param)
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args([])
args.parallelism = 4
args.no_dups = True
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name=job.job_id)
api = TuningRunManager(interface, args)
jobs = []
current_value = None
failed_jobs = 0
while failed_jobs < failed_jobs_threshold and not self._check_for_termination(
job):
remaining_jobs = []
for job_id, desired_result in jobs:
res = self._get_evaluation_job_result(job_id)
if res is not None:
if current_value is None or current_value > res + _THRESHOLD_EPS:
failed_jobs = 0
else:
failed_jobs += 1
result = Result(time=res)
api.report_result(desired_result, result)
else:
remaining_jobs.append((job_id, desired_result))
jobs = remaining_jobs
while len(jobs) < self.jobs_limit:
desired_result = api.get_next_desired_result()
if desired_result is None:
break
job_id = self._start_evaluation_job(
job, desired_result.configuration.data)
if job_id is None:
api.report_result(desired_result, Result(time=math.inf))
else:
jobs.append((job_id, desired_result))
if not jobs:
break
r = api.get_best_result()
if r is not None:
current_value = r.time
logging.debug('Opentuner current state: %s %s', r.time,
api.get_best_configuration())
time.sleep(5)
res = api.get_best_result().time
vars = api.get_best_configuration()
api.finish()
return res, vars
def tuning_loop():
report_delay = 200
last_time = time.time()
start_time = last_time
parser = argparse.ArgumentParser(parents = opentuner.argparsers())
parser.add_argument("--processes",
type = int,
help = "Number of Python threads available.")
parser.add_argument("--no-wait",
action = "store_true",
help = "Do not wait for requested results to generate more requests.")
args = parser.parse_args()
pool = ThreadPool(args.processes)
manipulator = ConfigurationManipulator()
for name in legup_parameters.parameters:
parameter_type = legup_parameters.parameter_type(name)
values = legup_parameters.parameter_values(name)
if parameter_type == int:
manipulator.add_parameter(IntegerParameter(name, values[0], values[1]))
elif parameter_type == bool:
manipulator.add_parameter(BooleanParameter(name))
elif parameter_type == Enum:
manipulator.add_parameter(EnumParameter(name, values))
else:
print("ERROR: No such parameter type \"{0}\"".format(name))
interface = DefaultMeasurementInterface(args = args,
manipulator = manipulator,
project_name = 'HLS-FPGAs',
program_name = 'legup-tuner',
program_version = '0.0.1')
manager = TuningRunManager(interface, args)
current_time = time.time()
computing_results = []
computed_results = []
desired_results = manager.get_desired_results()
while current_time - start_time < args.stop_after:
if args.no_wait:
if len(desired_results) != 0 or len(computing_results) != 0:
for desired_result in desired_results:
computing_results.append([desired_result,
pool.apply_async(get_wallclock_time,
(desired_result.configuration.data, ))])
for result in computing_results:
if result[1].ready() and result[0] not in computed_results:
cost = result[1].get()
manager.report_result(result[0], Result(time = cost))
computed_results.append(result)
for result in computed_results:
if result in computing_results:
computing_results.remove(result)
computed_results = []
else:
if len(desired_results) != 0:
cfgs = [dr.configuration.data for dr in desired_results]
results = pool.map_async(get_wallclock_time, cfgs).get(timeout = None)
for dr, result in zip(desired_results, results):
manager.report_result(dr, Result(time = result))
desired_results = manager.get_desired_results()
current_time = time.time()
if (current_time - last_time) >= report_delay:
log_intermediate(current_time - start_time, manager)
last_time = current_time
current_time = time.time()
log_intermediate(current_time - start_time, manager)
save_final_configuration(manager.get_best_configuration())
manager.finish()
def tuning_loop():
report_delay = 5
last_time = time.time()
start_time = last_time
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
parser.add_argument("--processes",
type=int,
help="Number of Python threads available.")
parser.add_argument(
"--no-wait",
action="store_true",
help="Do not wait for requested results to generate more requests.")
args = parser.parse_args()
pool = ThreadPool(args.processes)
manipulator = ConfigurationManipulator()
for name in legup_parameters.parameters:
parameter_type = legup_parameters.parameter_type(name)
values = legup_parameters.parameter_values(name)
if parameter_type == int:
manipulator.add_parameter(
IntegerParameter(name, values[0], values[1]))
elif parameter_type == bool:
manipulator.add_parameter(BooleanParameter(name))
elif parameter_type == Enum:
manipulator.add_parameter(EnumParameter(name, values))
else:
print("ERROR: No such parameter type \"{0}\"".format(name))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='HLS-FPGAs',
program_name='legup-tuner',
program_version='0.0.1')
manager = TuningRunManager(interface, args)
current_time = time.time()
computing_results = []
computed_results = []
desired_results = manager.get_desired_results()
while current_time - start_time < args.stop_after:
if args.no_wait:
if len(desired_results) != 0 or len(computing_results) != 0:
for desired_result in desired_results:
computing_results.append([
desired_result,
pool.apply_async(get_wallclock_time,
(desired_result.configuration.data, ))
])
for result in computing_results:
if result[1].ready() and result[0] not in computed_results:
cost = result[1].get()
manager.report_result(result[0], Result(time=cost))
computed_results.append(result)
for result in computed_results:
if result in computing_results:
computing_results.remove(result)
computed_results = []
else:
if len(desired_results) != 0:
cfgs = [dr.configuration.data for dr in desired_results]
results = pool.map_async(get_wallclock_time,
cfgs).get(timeout=None)
for dr, result in zip(desired_results, results):
manager.report_result(dr, Result(time=result))
desired_results = manager.get_desired_results()
current_time = time.time()
if (current_time - last_time) >= report_delay:
log_intermediate(current_time - start_time, manager)
last_time = current_time
current_time = time.time()
log_intermediate(current_time - start_time, manager)
save_final_configuration(manager.get_best_configuration())
manager.finish()
class PostgresTuner():
"""
An OpenTuner interface for Postgres indexing
OpenTuner API usage:
- define a parameter search space for parameters to tune.
- define a run loop, in which:
- OpenTuner returns a configuration, a particular assignment of parameters in search space
- we execute and evaluate that configuration on the system
- and return the reward to OpenTuner, refining the search
Search space:
1st:
- parameter per action taken for candidate index column (allow 3 index columns per index) per index (allow index per query)
- size... this is (up to) 3 candidate index columns per query * 100 episodes * 20 queries per episodes = 6000
2nd:
- parameter per action taken for candidate index column per index / indexing decision in an episode
- size? 3 * 20
- idea?
- analogous to how RL agent works. exposed to 20 queries per episode.
- and evaluate on a random subset of set of train queries
Sourced from:
- lift/case_studies/{mongodb,mysql}/
- https://github.com/jansel/opentuner/blob/master/examples/py_api/api_example.py
"""
def __init__(self, train_queries, test_queries, experiment_config,
result_dir, system_environment):
"""
Args:
train_queries (list of queries (SQLQuery queries))
test_queries (list of queries (SQLQuery queries))
result_dir (str)
system (PostgresSystemEnvironment): encapsulates environment
"""
self.logger = logging.getLogger(__name__)
self.logger.info('Setting up OpenTuner...')
self.train_queries = train_queries
self.test_queries = test_queries
self.experiment_config = experiment_config
self.max_size = experiment_config['max_size']
self.size_weight = experiment_config['size_weight']
self.max_runtime = experiment_config['max_runtime']
self.runtime_weight = experiment_config['runtime_weight']
self.reward_penalty = experiment_config['reward_penalty']
self.system_environment = system_environment
self.result_dir = result_dir
# 2nd search space
self.n_idxs = experiment_config['n_queries_per_episode']
self.n_cols_per_idx = 3 # TODO hardcode
self.tbls = experiment_config['tables']
self.n_idxs_per_tbl = int(self.n_idxs / len(self.tbls))
# maps tbls to mapping of tbl's column indices to tbl's columns, per search space representation
self.tbl_2_col_idx_2_col = {}
for tbl in self.tbls:
self.tbl_2_col_idx_2_col[tbl] = {}
for col_idx, col in enumerate(tpch_table_columns[tbl].keys()):
self.tbl_2_col_idx_2_col[tbl][col_idx +
1] = col # + 1 b/c 0 is noop
# api
sys.argv = [sys.argv[0]] # opentuner expects own args
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
args = parser.parse_args()
manipulator = self.build_search_space()
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator)
self.api = TuningRunManager(interface, args)
def build_search_space(self):
"""
Set up search space via ConfigurationManipulator, which is responsible for choosing configurations across runs,
where a configuration is a particular assignment of the parameters.
Tightly coupled with agent/system representations used in src.
Repeating from above:
*initial* search space
- parameter per action (candidate index column in index, where candidate index column is one of the query columns)
per query
- see commit 8f94c4b
*updated* search space
- parameter per action (same, but here candidate index column is not one of query columns but any of columns in query table)
per allowed index / indexing decision
so, at risk of redundancy, an "action" refers to the particular assignment of a parameter, which is
an integer indicating noop or a column for a candidate index column
"""
self.logger.info('Building OpenTuner search space...')
self.idx_2_idx_cols = {
} # store candidate index columns (OpenTuner operates on this) with candidate index (system operates on this)
self.idx_2_tbl = {}
manipulator = ConfigurationManipulator()
for tbl in self.tbls:
n_cols_per_tbl = len(tpch_table_columns[tbl])
for candidate_idx in range(self.n_idxs_per_tbl):
idx_id = "tbl_{}_idx_{}".format(tbl, candidate_idx)
idx_col_ids = []
for candidate_idx_col in range(self.n_cols_per_idx):
idx_col_id = idx_id + "_idx_col_{}".format(
candidate_idx_col)
idx_col_ids.append(idx_col_id)
manipulator.add_parameter(
IntegerParameter(idx_col_id, 0, n_cols_per_tbl))
self.idx_2_idx_cols[idx_id] = idx_col_ids
self.idx_2_tbl[idx_id] = tbl
self.logger.info("... actions are: {}".format(self.idx_2_idx_cols))
return manipulator
def run(self, n_iterations):
"""
Runs OpenTuner search
run loop - https://github.com/jansel/opentuner/blob/master/examples/py_api/api_example.py
"desired result" vs "result" - http://groups.csail.mit.edu/commit/papers/2014/ansel-pact14-opentuner.pdf
"""
self.logger.info('Running OpenTuner...')
# search
for i in range(n_iterations):
self.logger.info('iteration {}/{}...'.format(i + 1, n_iterations))
start = time.time()
desired_result = self.api.get_next_desired_result()
configuration = desired_result.configuration.data
reward, _ = self.act(configuration)
result = opentuner.resultsdb.models.Result(time=-1 * reward)
self.api.report_result(desired_result, result)
self.logger.info('...received reward {} after {} seconds'.format(
reward,
time.time() - start))
# best from search
best = self.api.get_best_configuration()
self.eval_best(best)
self.system_environment.reset()
def act(self, cfg):
"""
Get reward for current configuration i.e. recommended index.
Args:
cfg: current configuration returned by OpenTuner search, maps parameter to parameter value
"""
self.system_environment.reset()
context = []
episode_reward = 0
# for each candidate index, extract decisions for candidate index columns
for idx_id, idx_col_ids in self.idx_2_idx_cols.items():
# get integer (i.e. integer value of IntegerParameter)
actions = []
for idx_col_id in idx_col_ids:
action = cfg[idx_col_id]
actions.append(action)
# get corresponding attribute for integer
system_action = []
tbl = self.idx_2_tbl[idx_id]
for action in actions:
if action != 0: # noop
system_action.append(self.tbl_2_col_idx_2_col[tbl][action])
#
# execute
#
self.system_environment.act(dict(index=system_action, table=tbl))
context.append(system_action)
#
# evaluate on a randomly selected subset of workload
#
train_query_sample = np.random.choice(self.train_queries,
size=max(
len(self.train_queries) / 10,
1)) # TODO add to config
for query in train_query_sample:
query_string, query_string_args = query.sample_query()
query_time = self.system_environment.execute(
query_string, query_string_args)
index_set_size, index_set = self.system_environment.system_status()
reward = self.system_to_agent_reward(
data=dict(runtime=query_time, index_size=index_set_size))
episode_reward += reward
# N.B. context can't be used(!)
return episode_reward, context
def eval_best(self, config):
self.logger.info('Evaluating best OpenTuner configuration...')
pdb.set_trace()
# create configuration
reward, context = self.act(config)
runs = self.experiment_config['n_executions']
runtimes = []
for query in self.test_queries:
runtimes_per_query = []
for _ in range(runs):
query_string, query_string_args = query.sample_query()
query_time = self.system_environment.execute(
query_string, query_string_args)
runtimes_per_query.append(query_time)
runtimes.append(runtimes_per_query)
index_set_size, index_set = self.system_environment.system_status()
# runtimes
np.savetxt(os.path.join(result_dir, 'test_query_times.txt'),
np.asarray(runtimes),
delimiter=',')
# index set, index set size
with open(os.path.join(result_dir, 'test_index_set_stats.csv'),
'wb') as f:
pickle.dump([index_set_size, index_set], f)
#
# Auxiliary
#
def system_to_agent_reward(self, data):
"""
Same as in Converter, but didn't want to have Schema / Converter here
"""
runtime, index_size = data["runtime"], data["index_size"]
reward = -(self.runtime_weight * runtime) - (self.size_weight *
index_size)
if runtime > self.max_runtime:
reward -= self.reward_penalty
if index_size > self.max_size:
reward -= self.reward_penalty
return reward
def __init__(self,
api_config,
techniques=DEFAULT_TECHNIQUES,
n_suggestions=1):
"""Build wrapper class to use opentuner optimizer in benchmark.
Parameters
----------
api_config : dict-like of dict-like
Configuration of the optimization variables. See API description.
techniques : iterable of strings
A list or tuple of techniques to use in opentuner. If the list
has only one technique, then that technique will be used. If the
list has multiple techniques a bandit over those techniques
will be used.
n_suggestions : int
Default number of suggestions to be made in parallel.
"""
AbstractOptimizer.__init__(self, api_config)
# Opentuner requires DesiredResult to reference suggestion when making
# its observation. x_to_dr maps the dict suggestion to DesiredResult.
self.x_to_dr = {}
# Keep last suggested x and repeat it whenever opentuner gives up.
self.dummy_suggest = None
"""Setting up the arguments for opentuner. You can see all possible
arguments using:
```
>>> import opentuner
>>> opentuner.default_argparser().parse_args(['-h'])
```
We only change a few arguments (other arguments are set to defaults):
* database = MEMORY_ONLY_DB: to use an in-memory sqlite database
* parallelism = n_suggestions: num of suggestions to give in parallel
* technique = techniques: a list of techniques to be used by opentuner
* print_params = False: to avoid opentuner from exiting after printing
param spaces
"""
args = Namespace(
bail_threshold=500,
database=MEMORY_ONLY_DB,
display_frequency=10,
generate_bandit_technique=False,
label=None,
list_techniques=False,
machine_class=None,
no_dups=False,
parallel_compile=False,
parallelism=n_suggestions,
pipelining=0,
print_params=False,
print_search_space_size=False,
quiet=False,
results_log=None,
results_log_details=None,
seed_configuration=[],
stop_after=None,
technique=techniques,
test_limit=5000,
)
# Setup some dummy classes required by opentuner to actually run.
manipulator = OpentunerOptimizer.build_manipulator(api_config)
interface = DMI(args=args, manipulator=manipulator)
self.api = TuningRunManager(interface, args)
def tuning_loop():
report_delay = 30
last_time = time.time()
start_time = last_time
iterations = 5
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
parser.add_argument("--processes",
type=int,
help="Number of Python threads available.")
parser.add_argument(
"--no-wait",
action="store_true",
help="Do not wait for requested results to generate more requests.")
parser.add_argument("--application", type=str, help="Application name.")
parser.add_argument("--verilog-file",
type=str,
help="Verilog file for the application.")
args = parser.parse_args()
pool = ThreadPool(args.processes)
manipulator = ConfigurationManipulator()
global application
global verilog_file
global application_path
global container_path
global host_path
global image_name
global script_name
global tuning_init
application = args.application
verilog_file = args.verilog_file
application_path = "/root/legup_src/legup-4.0/examples/chstone/{0}".format(
application)
container_path = "/root/legup_src/legup-4.0/examples/chstone/{0}/tuner".format(
application)
host_path = "/home/bruelp/legup-tuner/post_place_and_route/py"
image_name = "legup_quartus"
script_name = "measure.sh"
print(application, container_path, application_path)
for name in legup_parameters.parameters:
parameter_type = legup_parameters.parameter_type(name)
values = legup_parameters.parameter_values(name)
if parameter_type == int:
manipulator.add_parameter(
IntegerParameter(name, values[0], values[1]))
elif parameter_type == bool:
manipulator.add_parameter(BooleanParameter(name))
elif parameter_type == Enum:
manipulator.add_parameter(EnumParameter(name, values))
else:
print("ERROR: No such parameter type \"{0}\"".format(name))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='HLS-FPGAs',
program_name='legup-tuner',
program_version='0.0.1')
manager = TuningRunManager(interface, args)
current_time = time.time()
computing_results = []
computed_results = []
desired_results = manager.get_desired_results()
while current_time - start_time < args.stop_after:
if args.no_wait:
if len(desired_results) != 0 or len(computing_results) != 0:
for desired_result in desired_results:
computing_results.append([
desired_result,
pool.apply_async(get_wallclock_time,
(desired_result.configuration.data, ))
])
for result in computing_results:
if result[1].ready() and result[0] not in computed_results:
cost = result[1].get()
manager.report_result(result[0], Result(time=cost))
computed_results.append(result)
for result in computed_results:
if result in computing_results:
computing_results.remove(result)
computed_results = []
else:
if len(desired_results) != 0:
cfgs = [dr.configuration.data for dr in desired_results]
results = pool.map_async(get_wallclock_time,
cfgs).get(timeout=None)
for dr, result in zip(desired_results, results):
manager.report_result(
dr,
Result(time=result['value'],
cycles=result['cycles'],
fmax=result['fmax'],
LU=result['lu'],
pins=result['pins'],
regs=result['regs'],
block=result['block'],
ram=result['ram'],
dsp=result['dsp']))
desired_results = manager.get_desired_results()
current_time = time.time()
if (current_time - last_time) >= report_delay:
log_intermediate(current_time - start_time, manager)
last_time = current_time
current_time = time.time()
log_intermediate(current_time - start_time, manager)
save_final_configuration(manager.get_best_configuration())
manager.finish()
