def setup(self):
"""Setup the cluster of hosts. Depending on the engine parameters it
will bootstrap hadoop directly or deploy a given environment.
"""
self.hosts = get_oar_job_nodes(self.oar_job_id, self.frontend)
if self.use_kadeploy:
(deployed, undeployed) = self.deploy_nodes()
return (len(deployed) != 0)
else:
if not self.hc:
self.hc = HadoopCluster(self.hosts)
self.hc.bootstrap(self.hadoop_tar_file)
return True
def run(self):
"""Execute a test suite. The execution workflow is as follows:
1. Parse command-line arguments.
2. Define the parameters of the tests from the specified configuration
file. Generate all the combination to test from the given parameters.
3. Consume the combinations.
3.1. Setup the cluster if it has not been done (first time or after a
reservation ends.
3.2. Load the dataset into the Hadoop cluster.
3.3. Perform the experiments corresponding to the combinations linked
to the loaded dataset.
4. Clean all resources.
"""
# Get parameters
self.cluster = self.args[0]
self.n_nodes = int(self.args[1])
self.config_file = self.args[2]
self.site = get_cluster_site(self.cluster)
if not os.path.exists(self.config_file):
logger.error("Params file " + self.config_file + " does not exist")
sys.exit(1)
# Set oar job id
if self.options.oar_job_id:
self.oar_job_id = self.options.oar_job_id
else:
self.oar_job_id = None
# Main
try:
# Creation of the main iterator used for the first control loop.
self.define_parameters()
job_is_dead = False
# While they are combinations to treat
while len(self.sweeper.get_remaining()) > 0:
# SETUP
# If no job, we make a reservation and prepare the hosts for the
# experiments
if job_is_dead or self.oar_job_id is None:
self.make_reservation()
success = self.setup()
if not success:
break
else:
self.hosts = get_oar_job_nodes(self.oar_job_id,
self.frontend)
if not self.hc:
self.hc = HadoopCluster(self.hosts)
# SETUP FINISHED
# Getting the next combination (which requires a ds deployment)
comb = self.sweeper.get_next()
self.raw_comb = comb.copy()
self.comb = comb
self.prepare_dataset(comb)
self.xp_wrapper(comb)
# subloop over the combinations that use the same dataset
while True:
newcomb = self.sweeper.get_next(
lambda r: filter(self._uses_same_ds, r))
if newcomb:
self.raw_comb = newcomb.copy()
try:
self.xp_wrapper(newcomb)
except:
break
else:
break
if get_oar_job_info(self.oar_job_id,
self.frontend)['state'] == 'Error':
job_is_dead = True
finally:
if self.oar_job_id is not None:
if not self.options.keep_alive:
pass
logger.info('Deleting job')
oardel([(self.oar_job_id, self.frontend)])
else:
logger.info('Keeping job alive for debugging')
# Clean cluster
if self.hc:
if self.hc.initialized:
self.hc.clean()
# Close summary files
if self.summary_file:
self.summary_file.close()
if self.ds_summary_file:
self.ds_summary_file.close()
