def manipulator(self):
"""
Returns an object that represents the parameters that can be tuned.
"""
manipulator = ConfigurationManipulator()
if 'ExhaustiveSearch' in self.args.technique:
# manipulator.add_parameter(EnumParameter("CLOCK", ['0', '1', '2', '3']))
# manipulator.add_parameter(EnumParameter("KERNEL_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(LogIntegerParameter("CONVOLVERS", 1, 16))
manipulator.add_parameter(
IntegerParameter("BIN_DENSE_PIPELINE", 1, 16))
# manipulator.add_parameter(IntegerParameter("FP_CONV_UNROLL", 1, 16))
else:
manipulator.add_parameter(
EnumParameter("KERNEL_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(
FloatParameter("CLOCK_UNCERTAINTY", 0, 100))
manipulator.add_parameter(
EnumParameter("DATA_MOVER_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(
EnumParameter("DATA_MOVER_SHARING", ['0', '1', '2', '3']))
manipulator.add_parameter(IntegerParameter("IMPL_STRATEGY", 0, 27))
manipulator.add_parameter(
LogIntegerParameter("CONVOLVERS", 1, 16, prior="inc"))
manipulator.add_parameter(
IntegerParameter("BIN_DENSE_PIPELINE", 1, 16, prior="dec"))
manipulator.add_parameter(
IntegerParameter("FP_CONV_UNROLL", 1, 16, prior="inc"))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
num_options = len(self.passes) - 1
# choose for a sequence of at most `max_passes` passes
for i in range(self.max_passes):
passNum = 'pass_' + str(i)
# some pass
manipulator.add_parameter(IntegerParameter(passNum, 0,
num_options))
# and whether to turn it on or off
manipulator.add_parameter(
EnumParameter('enable_' + passNum, [True, False]))
# choose from a set of knobs
for knob, minInt, maxInt in self.knobs:
manipulator.add_parameter(IntegerParameter(knob, minInt, maxInt))
manipulator.add_parameter(
EnumParameter('enable_' + knob, [True, False]))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
self.transformation(gl.spark_parameter)
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
IntegerParameter('BLOCK_SIZE', 1, 1))
# print 'in manipulator'
for flag in self.enum_param:
# print flag[0], flag[1]
manipulator.add_parameter(
EnumParameter(flag[0],
flag[1]))
for i in self.int_param:
manipulator.add_parameter(
IntegerParameter(i[0], int(i[1]), int(i[2])))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('opt_level', 0, 3))
for flag in GCC_FLAGS:
manipulator.add_parameter(
EnumParameter(flag, ['on', 'off', 'default']))
for param, min, max in GCC_PARAMS:
manipulator.add_parameter(IntegerParameter(param, min, max))
return manipulator
def manipulator(self):
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('nsup', 50, 300))
manipulator.add_parameter(IntegerParameter('nrel', 10, 40))
manipulator.add_parameter(IntegerParameter('colperm', 0, 4))
manipulator.add_parameter(IntegerParameter('lookahead', 0, 10))
manipulator.add_parameter(IntegerParameter('numthreads', 1, 10))
'''
Parameters for testing SuperLU
'''
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('batch_size', 5, 200))
manipulator.add_parameter(IntegerParameter('layer', 1, 20))
manipulator.add_parameter(IntegerParameter('neural', 50, 200))
#manipulator.add_parameter(IntegerParameter('max_iter', 1, 500))
#manipulator.add_parameter(FloatParameter('learning_rate_init', 0.0001, 0.01))
#manipulator.add_parameter(FloatParameter('power_t', 0.2, 0.8))
#manipulator.add_parameter(FloatParameter('alpha', 0.00001, 0.001))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
self.granularity = 1
assert (0 < self.granularity)
max_m = (160 + self.granularity - 1) / self.granularity
max_n = (160 + self.granularity - 1) / self.granularity
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
IntegerParameter("M", self.granularity, max_m))
manipulator.add_parameter(
IntegerParameter("N", self.granularity, max_n))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
for i, param in enumerate(self.params):
if param[0] == "None":
continue
if param[0] == "Int":
manipulator.add_parameter(
IntegerParameter(i, int(param[1]), int(param[2])))
continue
if param[0] == "Log":
manipulator.add_parameter(
LogIntegerParameter(i, int(param[1]), int(param[2])))
continue
if param[0] == "Pow":
manipulator.add_parameter(
PowerOfTwoParameter(i, 2**int(param[1]), 2**int(param[2])))
continue
if param[0] == "Switch":
manipulator.add_parameter(SwitchParameter(i, int(param[1])))
continue
if param[0] == "Bool":
manipulator.add_parameter(BooleanParameter(i))
continue
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('blockSize', 1, 10))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(IntegerParameter('TILE_SIZE', 2, 256))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
m_max = min(160, self.args.end)
n_max = min(160, self.args.end)
self.granularity = 1
assert(0 < self.granularity)
m_max = (m_max + self.granularity - 1) / self.granularity
n_max = (n_max + self.granularity - 1) / self.granularity
m_param = IntegerParameter("M", self.granularity, m_max)
n_param = IntegerParameter("N", self.granularity, n_max)
manipulator = ConfigurationManipulator()
manipulator.add_parameter(m_param)
manipulator.add_parameter(n_param)
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
self.granularity = 1
assert (0 < self.granularity)
m_max = (64 + self.granularity - 1) / self.granularity
n_max = (256 + self.granularity - 1) / self.granularity
k_max = (256 + self.granularity - 1) / self.granularity
m_param = IntegerParameter("M", self.granularity, m_max)
n_param = IntegerParameter("N", self.granularity, n_max)
k_param = IntegerParameter("K", self.granularity, k_max)
manipulator = ConfigurationManipulator()
manipulator.add_parameter(m_param)
manipulator.add_parameter(n_param)
manipulator.add_parameter(k_param)
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
self.dimset = libxsmm_utilities.load_mnklist(self.args.mnk, 0, -1)
self.granularity = 1
assert (0 < self.granularity)
m_max = (64 + self.granularity - 1) / self.granularity
n_max = (256 + self.granularity - 1) / self.granularity
k_max = (256 + self.granularity - 1) / self.granularity
m_param = IntegerParameter("M", self.granularity, m_max)
n_param = IntegerParameter("N", self.granularity, n_max)
k_param = IntegerParameter("K", self.granularity, k_max)
manipulator = ConfigurationManipulator()
manipulator.add_parameter(m_param)
manipulator.add_parameter(n_param)
manipulator.add_parameter(k_param)
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
# set the global flags needed for printing schedules
if self.args.enable_NUMA_tuning == 0:
self.enable_NUMA_tuning = False
if self.args.enable_parallel_tuning == 0:
self.enable_parallel_tuning = False
if self.args.enable_denseVertexSet_tuning == 0:
self.enable_denseVertexSet_tuning = False
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
EnumParameter('direction',
['SparsePush','DensePull', 'SparsePush-DensePull', 'DensePush-SparsePush']))
#'edge-aware-dynamic-vertex-parallel' not supported with the latest g++ cilk implementation
if self.enable_parallel_tuning:
manipulator.add_parameter(EnumParameter('parallelization',['dynamic-vertex-parallel']))
else:
manipulator.add_parameter(EnumParameter('parallelization', ['serial']))
manipulator.add_parameter(IntegerParameter('numSSG', 1, self.args.max_num_segments))
if self.enable_NUMA_tuning:
manipulator.add_parameter(EnumParameter('NUMA',['serial','static-parallel']))
if self.enable_denseVertexSet_tuning:
manipulator.add_parameter(EnumParameter('DenseVertexSet', ['boolean-array', 'bitvector']))
# adding new parameters for PriorityGraph (Ordered GraphIt)
manipulator.add_parameter(IntegerParameter('delta', 1, self.args.max_delta))
manipulator.add_parameter(
EnumParameter('bucket_update_strategy',
['eager_priority_update','eager_priority_update_with_merge', 'lazy_priority_update']))
return manipulator
def manipulator(self):
manipulator = ConfigurationManipulator()
for item in self.param:
param_type, param_name, param_range = item
if param_type == 'EnumParameter':
manipulator.add_parameter(EnumParameter(param_name, param_range))
elif param_type == 'IntegerParameter' :
manipulator.add_parameter(IntegerParameter(param_name, param_range[0], param_range[1]))
elif param_type == 'FloatParameter' :
manipulator.add_parameter(FloatParameter(param_name, param_range[0], param_range[1]))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
self.mintilesize = 2
self.granularity = 1
assert (0 < self.granularity)
minsize = max(self.mintilesize / self.granularity, 1)
maxsize = minsize + self.granularity
m_max = max(min(self.args.maxm, self.args.end), maxsize)
n_max = max(min(self.args.maxn, self.args.end), maxsize)
m_max = (m_max + self.granularity - 1) / self.granularity
n_max = (n_max + self.granularity - 1) / self.granularity
m_param = IntegerParameter("M", minsize, m_max)
n_param = IntegerParameter("N", minsize, n_max)
manipulator = ConfigurationManipulator()
manipulator.add_parameter(m_param)
manipulator.add_parameter(n_param)
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
# manipulator.add_parameter(
# IntegerParameter('blockSize', 8,16))
manipulator.add_parameter(IntegerParameter('opt_level', 0, 3))
manipulator.add_parameter(
EnumParameter('blockSize', ['8', '16', '32', '64', '128']))
return manipulator
def manipulator(self):
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
LogIntegerParameter("INIT_INTERVAL_HOR", 1, 16))
manipulator.add_parameter(
LogIntegerParameter("INIT_INTERVAL_VER", 1, 16))
manipulator.add_parameter(IntegerParameter("UNROLL_FACTOR_HOR", 1, 2))
manipulator.add_parameter(IntegerParameter("UNROLL_FACTOR_VER", 1, 2))
manipulator.add_parameter(
LogIntegerParameter("ARRAY_PARTITION_FACTOR", 1, 15))
manipulator.add_parameter(
EnumParameter("ACCELERATOR_1_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(
EnumParameter("ACCELERATOR_2_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(
FloatParameter("ACCELERATOR_1_UNCERTAINTY", 0, 100))
manipulator.add_parameter(
FloatParameter("ACCELERATOR_2_UNCERTAINTY", 0, 100))
manipulator.add_parameter(
EnumParameter("DATA_MOVER_CLOCK", ['0', '1', '2', '3']))
return manipulator
def manipulator(self):
self.expose_defines()
manipulator = ConfigurationManipulator()
params = self.get_parameters()
for line in params:
[param, type] = line.split()[3:5]
if type == 'integer':
start, end = line.split()[5:7]
manipulator.add_parameter(IntegerParameter(param, start, end))
else:
raise HLSTunerError(
"Unknown parameter type encountered in tuner parameter pragma."
)
return manipulator
def manipulator(self):
"""Define the search space by creating a
ConfigurationManipulator
"""
list_configs = []
list_configs = read_file_configs()
manipulator = ConfigurationManipulator()
print "Executing manipulator"
for param, mini, maxi in BLOCO_PARAMETROS:
#print "param: ", param, " mini: ", mini, " maxi: ", maxi
manipulator.add_parameter(IntegerParameter(param, mini, maxi))
# E preciso gerar as configuracoes validas com o run.c.
# manipulator.add_parameter(EnumParameter(param, [ 'gx:1024, gy:1, gz:1, bx:1, by:1, bz:1, ', 'gx:32, gy:32, gz:1, bx:1, by:1, bz:1, ' ]))
for param in BLOCO_PARAMETROS_CONFIGS:
manipulator.add_parameter(EnumParameter(param, list_configs))
return manipulator
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
# set the global flags needed for printing schedules
if self.args.enable_NUMA_tuning == 0:
self.enable_NUMA_tuning = False
if self.args.enable_parallel_tuning == 0:
self.enable_parallel_tuning = False
if self.args.enable_denseVertexSet_tuning == 0:
self.enable_denseVertexSet_tuning = False
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
EnumParameter('direction', [
'SparsePush', 'DensePull', 'SparsePush-DensePull',
'DensePush-SparsePush'
]))
if self.enable_parallel_tuning:
manipulator.add_parameter(
EnumParameter('parallelization', [
'dynamic-vertex-parallel',
'edge-aware-dynamic-vertex-parallel'
]))
else:
manipulator.add_parameter(
EnumParameter('parallelization', ['serial']))
manipulator.add_parameter(
IntegerParameter('numSSG', 1, self.args.max_num_segments))
if self.enable_NUMA_tuning:
manipulator.add_parameter(
EnumParameter('NUMA', ['serial', 'static-parallel']))
if self.enable_denseVertexSet_tuning:
manipulator.add_parameter(
EnumParameter('DenseVertexSet',
['boolean-array', 'bitvector']))
return manipulator
def parse_line(self, line, manipulator):
"""
Parse one line from configuration file
"""
line = line.split()
if (line[0] == 'parameter'):
if (line[1] == 'int'):
manipulator.add_parameter(IntegerParameter(line[2], line[3], line[4]))
elif (line[1] == 'float'):
manipulator.add_parameter(FloatParameter(line[2], line[3], line[4]))
elif (line[1] == 'enum'):
manipulator.add_parameter(EnumParameter(line[2], line[3:]))
else:
raise NameError('Incorrect Configuration file')
elif (line[0] == 'compile'):
self.gcc_cmd = ' '.join(line[1:])
elif (line[0] == 'run'):
self.run_cmd = ' '.join(line[1:])
else:
raise NameError('Incorrect Configuration file')
def manipulator(self):
m = ConfigurationManipulator()
for param in self.int_params:
param_value = self.int_params[param]
if param_value['max'] > 128:
m.add_parameter(
LogIntegerParameter(param, param_value['min'],
param_value['max']))
else:
m.add_parameter(
IntegerParameter(param, param_value['min'],
param_value['max']))
for param in self.power_of_two_params:
param_value = self.power_of_two_params[param]
m.add_parameter(
manipulator.PowerOfTwoParameter(param, param_value['min'],
param_value['max']))
return m
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
sys.stderr.write('AUTOTUNER: Create the design space\n')
# Check if manipulator has already been invoked
defineDesignSpace = len(self.designSpace) == 0
# Read the range of each dimension of the design space
inputFile = os.environ['RANGE_FILE']
ranges = utils.readFile(inputFile)
# Describe the design space to opentuner
param = 0
manipulator = ConfigurationManipulator()
for elem in ranges:
# Check if there is a design space
if defineDesignSpace:
self.designSpace.append(elem)
# Check if the current dimension has a cardinality higher than 1
if elem > 1:
# Check the type of the parameter
paramType = param % 9
# Create the parameter
if paramType == 0:
# Should the loop be parallelized?
openTuner_param = SwitchParameter(str(param), elem)
elif paramType == 1:
# Unroll factor
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 2:
# Peel factor
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 3:
# Parallelization technique
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 4:
# Number of cores to dedicate to the current loop
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 5:
# DOALL parameter: chunk factor
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 6:
# HELIX parameter: should we fix the maximum number of sequential segments?
openTuner_param = SwitchParameter(str(param), elem)
elif paramType == 7:
# HELIX parameter: maximum number of sequential segments
openTuner_param = IntegerParameter(str(param), 0,
str(elem - 1))
elif paramType == 8:
# DSWP parameter: should we use queue packing?
openTuner_param = SwitchParameter(str(param), elem)
# Share the parameter to OpenTuner
manipulator.add_parameter(openTuner_param)
param += 1
# Load outputDistortion module if possible
benchmark = os.environ['BENCHMARK_NAME']
self.outputDistortionModule = None
try:
self.outputDistortionModule = importlib.import_module(
'benchmark.' + benchmark + '.outputDistortion')
except ImportError:
self.outputDistortionModule = None
if defineDesignSpace:
sys.stderr.write('AUTOTUNER: Whole design space (' +
str(len(self.designSpace)) + ') = ' +
str(self.designSpace) + '\n')
return manipulator
def tune(args,
n,
eps,
delta,
skew,
m_range,
d_range,
tune_run=100,
spec_run=100,
save=None,
label=""):
results = AxProfTune.AxProfTune(
args,
{
"n": n,
"eps": eps,
"delta": delta,
"skew": skew
},
[
IntegerParameter("m", m_range[0], m_range[1]),
IntegerParameter("d", d_range[0], d_range[1]),
],
[1 - delta],
tune_run,
spec_run,
AxProf.zipfGenerator,
input_params,
runner,
spec,
acc_metric,
)
opt_config = results[0][1]
opt_m, opt_d = opt_config["m"], opt_config["d"]
if save is not None:
fd, fname = tempfile.mkstemp(
suffix=".json",
prefix="dp-n-{}-eps-{}-delta-{}-zipf-{}-".format(
n, eps, delta, skew),
dir=save,
)
f = os.fdopen(fd, "w")
f.write(
json.dumps({
"n": n,
"eps": eps,
"delta": delta,
"skew": skew,
"m_range": m_range,
"d_range": d_range,
"tune_run": tune_run,
"spec_run": spec_run,
"opt_m": opt_m,
"opt_d": opt_d,
"label": label,
}))
f.close()
print("saved:", fname)
return opt_m, opt_d
def manipulator(self):
"""
Returns an object that represents the parameters that can be tuned.
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(BooleanParameter("PIPELINE_SUB"))
manipulator.add_parameter(BooleanParameter("PIPELINE_ADDKEY"))
manipulator.add_parameter(BooleanParameter("PIPELINE_CPKEY"))
manipulator.add_parameter(BooleanParameter("PIPELINE_MIX"))
manipulator.add_parameter(BooleanParameter("PIPELINE_EXP1"))
manipulator.add_parameter(BooleanParameter("PIPELINE_EXP2"))
manipulator.add_parameter(BooleanParameter("PIPELINE_ECB1"))
manipulator.add_parameter(BooleanParameter("PIPELINE_ECB2"))
manipulator.add_parameter(BooleanParameter("PIPELINE_ECB3"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_SUB", 1, 16, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_ADDKEY", 1, 16, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_CPKEY", 1, 16, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_MIX", 1, 4, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_EXP1", 1, 3, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_EXP2", 1, 3, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_ECB1", 1, 32, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_ECB2", 1, 8, prior="inc"))
manipulator.add_parameter(
IntegerParameter("UNROLL_FACTOR_ECB3", 1, 13, prior="inc"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_SUB", 1, 16, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_ADDKEY", 1, 16, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_CPKEY", 1, 16, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_MIX", 1, 4, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_EXP1", 1, 3, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_EXP2", 1, 3, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_ECB1", 1, 32, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_ECB2", 1, 8, prior="dec"))
manipulator.add_parameter(
IntegerParameter("PIPELINE_II_ECB3", 1, 13, prior="dec"))
manipulator.add_parameter(
EnumParameter("KERNEL_CLOCK", ['0', '1', '2', '3']))
manipulator.add_parameter(
EnumParameter("DATA_MOVER_CLOCK", ['0', '1', '2', '3']))
return manipulator
def integer(self, name, min_value, max_value):
if self.first(name):
self.manipulator.add_parameter(
IntegerParameter(name, min_value, max_value))
return min_value
def addParams(self, manipulator):
manipulator.add_parameter(IntegerParameter(self.name, 0, self.max_pts))
def tuning_loop():
report_delay = 5
last_time = time.time()
start_time = last_time
parser = argparse.ArgumentParser(parents=opentuner.argparsers())
parser.add_argument("--processes",
type=int,
help="Number of Python threads available.")
parser.add_argument(
"--no-wait",
action="store_true",
help="Do not wait for requested results to generate more requests.")
args = parser.parse_args()
pool = ThreadPool(args.processes)
manipulator = ConfigurationManipulator()
for name in legup_parameters.parameters:
parameter_type = legup_parameters.parameter_type(name)
values = legup_parameters.parameter_values(name)
if parameter_type == int:
manipulator.add_parameter(
IntegerParameter(name, values[0], values[1]))
elif parameter_type == bool:
manipulator.add_parameter(BooleanParameter(name))
elif parameter_type == Enum:
manipulator.add_parameter(EnumParameter(name, values))
else:
print("ERROR: No such parameter type \"{0}\"".format(name))
interface = DefaultMeasurementInterface(args=args,
manipulator=manipulator,
project_name='HLS-FPGAs',
program_name='legup-tuner',
program_version='0.0.1')
manager = TuningRunManager(interface, args)
current_time = time.time()
computing_results = []
computed_results = []
desired_results = manager.get_desired_results()
while current_time - start_time < args.stop_after:
if args.no_wait:
if len(desired_results) != 0 or len(computing_results) != 0:
for desired_result in desired_results:
computing_results.append([
desired_result,
pool.apply_async(get_wallclock_time,
(desired_result.configuration.data, ))
])
for result in computing_results:
if result[1].ready() and result[0] not in computed_results:
cost = result[1].get()
manager.report_result(result[0], Result(time=cost))
computed_results.append(result)
for result in computed_results:
if result in computing_results:
computing_results.remove(result)
computed_results = []
else:
if len(desired_results) != 0:
cfgs = [dr.configuration.data for dr in desired_results]
results = pool.map_async(get_wallclock_time,
cfgs).get(timeout=None)
for dr, result in zip(desired_results, results):
manager.report_result(dr, Result(time=result))
desired_results = manager.get_desired_results()
current_time = time.time()
if (current_time - last_time) >= report_delay:
log_intermediate(current_time - start_time, manager)
last_time = current_time
current_time = time.time()
log_intermediate(current_time - start_time, manager)
save_final_configuration(manager.get_best_configuration())
manager.finish()
def manipulator(self):
"""
Define the search space by creating a
ConfigurationManipulator
"""
manipulator = ConfigurationManipulator()
manipulator.add_parameter(
EnumParameter('hibench.yarn.executor.cores',
['1', '4', '8', '16', '24', '48']))
manipulator.add_parameter(
EnumParameter('hibench.yarn.executor.num',
['1', '2', '3', '6', '12', '48']))
manipulator.add_parameter(IntegerParameter('spark.task.cpus', 1, 4))
manipulator.add_parameter(
EnumParameter('spark.shuffle.compress', ['1', '0']))
manipulator.add_parameter(
EnumParameter('spark.memory.fraction',
['0.25', '0.4', '0.5', '0.6', '0.7', '0.8']))
manipulator.add_parameter(EnumParameter('spark.serializer',
['0', '1']))
manipulator.add_parameter(
EnumParameter('spark.scheduler.maxRegisteredResourcesWaitingTime',
['10', '15', '25', '30', '40', '60']))
manipulator.add_parameter(
EnumParameter(
'spark.default.parallelism',
['8', '15', '25', '35', '50', '-1'
])) #-1 represents default value of config and we need it
manipulator.add_parameter(
EnumParameter('spark.sql.shuffle.partitions',
['75', '100', '150', '200', '250', '400']))
manipulator.add_parameter(
EnumParameter('spark.cleaner.periodicGC.interval',
['5', '10', '15', '30', '35', '60']))
manipulator.add_parameter(
EnumParameter('spark.io.compression.lz4.blockSize',
['8', '16', '32', '40', '64', '80']))
manipulator.add_parameter(
EnumParameter('spark.memory.storageFraction',
['0.3', '0.35', '0.4', '0.5', '0.6', '0.7']))
manipulator.add_parameter(
EnumParameter('spark.yarn.am.memory',
['256', '400', '512', '750', '1000', '1500']))
manipulator.add_parameter(
EnumParameter('spark.scheduler.revive.interval',
['1000', '1500', '2000', '2500', '3000', '4000']))
manipulator.add_parameter(
EnumParameter('spark.locality.wait.process',
['1500', '2000', '2500', '3000', '4000', '6000']))
manipulator.add_parameter(
EnumParameter('spark.shuffle.sort.bypassMergeThreshold',
['75', '100', '150', '200', '250', '400']))
manipulator.add_parameter(
EnumParameter('spark.shuffle.io.preferDirectBufs', ['1', '0']))
manipulator.add_parameter(
EnumParameter('spark.task.maxFailures',
['1', '2', '3', '4', '6', '8']))
manipulator.add_parameter(
EnumParameter('spark.files.openCostInBytes', [
'1000000', '2000000', '3500000', '4194304', '7000000',
'9000000'
]))
manipulator.add_parameter(
EnumParameter('spark.shuffle.file.buffer',
['16', '28', '32', '45', '64', '80']))
manipulator.add_parameter(
EnumParameter('spark.cleaner.referenceTracking.blocking',
['1', '0']))
manipulator.add_parameter(
EnumParameter('spark.kryoserializer.buffer.max',
['24', '32', '50', '64', '90', '128']))
manipulator.add_parameter(
EnumParameter('spark.executor.memory', [
'600', '1000', '2400', '4800', '9600', '19200', '54000',
'108000'
]))
return manipulator
