def keystone_exp():
sweeper = ParamSweeper(SWEEPER_DIR,
sweeps=sweep({
'db': DATABASES,
'delay': DELAYS,
'db-nodes': CLUSTER_SIZES
}))
while sweeper.get_remaining():
combination = sweeper.get_next()
logging.info("Treating combination %s" % pformat(combination))
try:
# Setup parameters
conf = copy.deepcopy(CONF) # Make a deepcopy so we can run
# multiple sweeps in parallels
conf['g5k']['resources']['machines'][0]['nodes'] = combination[
'db-nodes']
conf['tc']['constraints'][0][
'delay'] = "%sms" % combination['delay']
db = combination['db']
locality = False
xp_name = "%s-%s-%s" % (db, combination['db-nodes'],
combination['delay'])
# Let's get it started hun!
j.deploy(conf, db, locality, xp_name)
j.openstack(db)
j.emulate(conf['tc'])
j.rally(SCENARIOS, "keystone")
j.backup()
# Everything works well, mark combination as done
sweeper.done(combination)
logging.info("End of combination %s" % pformat(combination))
except Exception as e:
# Oh no, something goes wrong! Mark combination as cancel for
# a later retry
logging.error("Combination %s Failed with message %s" %
(pformat(combination), e))
sweeper.cancel(combination)
finally:
teardown()
def define_parameters(self):
"""Create the iterator that contains the parameters to be explored."""
config = ConfigParser()
config.readfp(open(self.config_file))
# TEST PARAMETERS
self.__define_test_parameters(config)
# DATASET PARAMETERS
self.__define_ds_parameters(config)
# EXPERIMENT PARAMETERS
xp_parameters_names = config.options("xp_parameters")
self.xp_parameters = {}
for pn in xp_parameters_names:
pv = config.get("xp_parameters", pn).split(",")
self.xp_parameters[pn] = [v.strip() for v in pv]
# GLOBAL
self.parameters = {}
self.parameters.update(self.ds_parameters)
self.parameters.update(self.xp_parameters)
# MACROS
self.macro_manager.add_ds_params(self.ds_parameters)
self.macro_manager.add_xp_params(self.xp_parameters)
for pn in self.parameters:
for pv in self.parameters[pn]:
macros = re.findall('\$\{([^}]*)\}',
str(pv)) # TODO: filter out non-strings
for m in macros:
self.macro_manager.add_dependency(m, pn)
self.macro_manager.sort_macros()
# SUMMARY FILES
# Xp summary
self.summary_file = open(self.summary_file_name, "w")
self.summary_props = []
self.summary_props.extend(self.ds_parameters.keys())
self.summary_props.extend(self.xp_parameters.keys())
header = "comb_id, job_id"
for pn in self.summary_props:
header += ", " + str(pn)
self.summary_file.write(header + "\n")
self.summary_file.flush()
# Ds summary
self.ds_summary_file = open(self.ds_summary_file_name, "w")
header = "ds_id, ds_class, ds_class_properties"
self.ds_summary_file.write(header + "\n")
self.ds_summary_file.flush()
# PRINT PARAMETERS
print_ds_parameters = {}
print_ds_parameters.update(self.ds_parameters)
print_ds_parameters["ds.config"] = self.ds_config
logger.info("Dataset parameters: " + str(print_ds_parameters))
logger.info("Experiment parameters: " + str(self.xp_parameters))
self.sweeper = ParamSweeper(os.path.join(self.result_dir, "sweeps"),
sweep(self.parameters))
logger.info(
'Number of parameters combinations %s, '
'Number of repetitions %s', len(self.sweeper.get_remaining()),
self.num_repetitions)
