def launch_jt(self):
"""Create and monitor the instance running the Jobtracker.
This also blocks and waits for the NameNode, then starts the JobTracker and
starts the agent that monitors Hadoop.
"""
# Keep this on the spawn_scheduler because it's hi-pri
if not self.spawn_instance(cfg.hadoop_jobtracker,
'hadoop/jobtracker_snitch.py'):
self.update_state('cluster', CluserState.BROKEN)
return
if not self.monitor_instance(cfg.hadoop_jobtracker,
InstanceState.SNITCH_READY):
self.update_state('cluster', CluserState.BROKEN)
return
with self.cv:
while self.instances[
cfg.hadoop_namenode] != InstanceState.HADOOP_READY:
self.cv.wait()
util.checked_do(cfg.hadoop_jobtracker, '/start', {})
self.update_state(cfg.hadoop_jobtracker, InstanceState.HADOOP_READY)
# Fork off and start our Java Hadoop monitor
util.bg_exec(
['java', '-cp', 'hadoop-tools.jar', 'com.google.HadoopMonitor'],
'/home/hadoop/monitor_log')
with self.cv:
self.cv.notifyAll()
def launch_jt(self):
"""Create and monitor the instance running the Jobtracker.
This also blocks and waits for the NameNode, then starts the JobTracker and
starts the agent that monitors Hadoop.
"""
# Keep this on the spawn_scheduler because it's hi-pri
if not self.spawn_instance(cfg.hadoop_jobtracker,
'hadoop/jobtracker_snitch.py'):
self.update_state('cluster', CluserState.BROKEN)
return
if not self.monitor_instance(cfg.hadoop_jobtracker,
InstanceState.SNITCH_READY):
self.update_state('cluster', CluserState.BROKEN)
return
with self.cv:
while self.instances[cfg.hadoop_namenode] != InstanceState.HADOOP_READY:
self.cv.wait()
util.checked_do(cfg.hadoop_jobtracker, '/start', {})
self.update_state(cfg.hadoop_jobtracker, InstanceState.HADOOP_READY)
# Fork off and start our Java Hadoop monitor
util.bg_exec(
['java', '-cp', 'hadoop-tools.jar', 'com.google.HadoopMonitor'],
'/home/hadoop/monitor_log'
)
with self.cv:
self.cv.notifyAll()
def start_job():
"""Downloads a JAR locally and submits a MapReduce job."""
common_snitch.authorize()
jar = bottle.request.forms.get('jar')
job_args = map(str, json.loads(bottle.request.forms.get('args')))
local_jobdir = tempfile.mkdtemp()
local_jar = os.path.join(local_jobdir, os.path.basename(jar))
get_file(jar, local_jar)
job_name = '{0}_{1}'.format(local_jar, uuid.uuid1())
util.bg_exec([cfg.hadoop_bin + 'hadoop', 'jar', local_jar] + job_args,
'/home/hadoop/log_job_{0}'.format(os.path.basename(job_name)))
return cfg.ok_reply
def start_job():
"""Downloads a JAR locally and submits a MapReduce job."""
common_snitch.authorize()
jar = bottle.request.forms.get('jar')
job_args = map(str, json.loads(bottle.request.forms.get('args')))
local_jobdir = tempfile.mkdtemp()
local_jar = os.path.join(local_jobdir, os.path.basename(jar))
get_file(jar, local_jar)
job_name = '{0}_{1}'.format(local_jar, uuid.uuid1())
util.bg_exec([cfg.hadoop_bin + 'hadoop', 'jar', local_jar] + job_args,
'/home/hadoop/log_job_{0}'.format(
os.path.basename(job_name)))
return cfg.ok_reply
