def __init__(self, master, dir_home):
# User defined
self._app = None # use self.set_app()
self.hdfs_file = None
self.slave_amounts = []
self.slave_amount = 0
self.dfs_replications = {}
self.repetitions = 0
self.repetition = 0
# Subclass
self._pool = None
self._input_host = None
self._input_file = None
# Shared
self._dir_exp = dir_home + '/exp04'
dir_output = self._dir_exp + '/outputs'
self._slaves = []
self._logger = ExpyLogger.getLogger('experiment')
# Spark
dir_spark = dir_home + '/spark/spark'
self._spark = Spark(home=dir_spark, master=master)
self._spark_ex = self._spark.executor
self._output_file = dir_output + \
'/spark-submit/slaves{:02d}_rep{:02d}_{}.txt'
# HDFS
dir_hadoop = dir_home + '/hadoop/hadoop'
self._hdfs = HDFS(home=dir_hadoop, name_node=master)
self._hdfs_ex = self._hdfs.executor
self._dir_hadoop_tmp = dir_home + '/hadoop-tmp'
self._blocks_file = dir_output + \
'/hdfs-blocks/slaves{:02d}_rep{:02d}.txt'
class Experiment:
def __init__(self, master, dir_home):
# User defined
self._app = None # use self.set_app()
self.hdfs_file = None
self.slave_amounts = []
self.slave_amount = 0
self.dfs_replications = {}
self.repetitions = 0
self.repetition = 0
# Subclass
self._pool = None
self._input_host = None
self._input_file = None
# Shared
self._dir_exp = dir_home + '/exp04'
dir_output = self._dir_exp + '/outputs'
self._slaves = []
self._logger = ExpyLogger.getLogger('experiment')
# Spark
dir_spark = dir_home + '/spark/spark'
self._spark = Spark(home=dir_spark, master=master)
self._spark_ex = self._spark.executor
self._output_file = dir_output + \
'/spark-submit/slaves{:02d}_rep{:02d}_{}.txt'
# HDFS
dir_hadoop = dir_home + '/hadoop/hadoop'
self._hdfs = HDFS(home=dir_hadoop, name_node=master)
self._hdfs_ex = self._hdfs.executor
self._dir_hadoop_tmp = dir_home + '/hadoop-tmp'
self._blocks_file = dir_output + \
'/hdfs-blocks/slaves{:02d}_rep{:02d}.txt'
def set_app(self, basename):
self._app = '{}/{}'.format(self._dir_exp, basename)
def _set_slaves(self, slaves):
self._slaves = slaves
self._systems_do(lambda s: s.set_slaves(slaves))
def run(self):
self._stop()
self._restart_history_server()
slave_amounts = [n for n in self.slave_amounts
if n >= self.slave_amount]
for self.slave_amount in slave_amounts:
self._get_slaves()
for self.repetition in range(self.repetition, self.repetitions):
self._hdfs_ex.add_barrier()
self._spark_ex.add_barrier()
self._run_once()
self.repetition = 0
self._mobile()
def _restart_history_server(self):
self._spark.stop_history_server()
self._spark_ex.add_barrier()
self._spark.start_history_server()
def _mobile(self):
body = 'Slave amount {:d} finished.'.format(self.slave_amount)
Pushbullet().send_note('Experiment USP', body)
def _run_once(self):
self._start()
self._hdfs_ex.add_barrier()
self._save_blocks_info()
self._submit_app()
self._stop()
def _submit_app(self):
"""Block while app is running."""
stdout = self._output_file.format(self.slave_amount, self.repetition,
'stdout')
stderr = self._output_file.format(self.slave_amount, self.repetition,
'stderr')
self._wait()
self._spark.submit(self._app, stdout, stderr)
self._spark_ex.wait()
def _stop(self):
"""Stop systems and clean temporary files."""
self._systems_do(lambda s: s.stop())
self._spark_ex.add_barrier()
self._spark.clean_tmp()
def finish(self):
self._systems_do(lambda s: s.executor.add_barrier())
self._spark.stop()
self._spark_ex.add_barrier()
self._spark.clean_tmp()
self._hdfs.stop()
self._pool.stop()
self._pool.wait()
self._wait()
def _get_slaves(self):
"""When a VM is started, its logs are deleted."""
# Hostname resolutions fail while starting VMs, so we wait.
self._wait()
slaves = self._pool.get(self.slave_amount)
self._set_slaves(slaves)
if self.repetition == 0:
# Bootstrap recently started nodes
self._bootstrap_hosts(self._pool.last_started)
def _start(self):
self._spark.start()
# Format namenode
if self.repetition == 0:
self._clean_hdfs()
else:
self._hdfs.start()
self._wait()
time.sleep(60)
def _clean_hdfs(self):
self._hdfs.format(self._dir_hadoop_tmp)
self._hdfs_ex.add_barrier()
self._hdfs.start()
self._hdfs_ex.wait()
time.sleep(60)
self._upload_input()
def _save_blocks_info(self):
output = self._blocks_file.format(len(self._slaves), self.repetition)
self._hdfs.save_block_locations(self.hdfs_file, output)
def _upload_input(self):
self._logger.info('%d slaves, repetition %d.', self.slave_amount,
self.repetition)
pipe = 'pbzip2 -dc ' + self._input_file
repl = self.dfs_replications[self.slave_amount]
self._hdfs.put_from_pipe(self._input_host, pipe, self.hdfs_file, repl)
def _bootstrap_hosts(self, hosts):
self._systems_do(lambda s: s.clean_logs(hosts))
cmd = 'VM={} make --quiet bootstrap-vm'
executor = Executor()
for host in hosts:
executor.run(Shell(cmd.format(host), 'bootstrap ' + host))
executor.shutdown() # blocking
def _wait(self):
self._systems_do(lambda s: s.executor.wait())
def _systems_do(self, func):
func(self._spark)
func(self._hdfs)