def submit_job(self, jar, job_args): if self.state == CluserState.READY: util.checked_do(cfg.hadoop_jobtracker, '/job/start', {'jar': jar, 'args': json.dumps(job_args)}) return True else: return False
def launch_jt(self): """Create and monitor the instance running the Jobtracker. This also blocks and waits for the NameNode, then starts the JobTracker and starts the agent that monitors Hadoop. """ # Keep this on the spawn_scheduler because it's hi-pri if not self.spawn_instance(cfg.hadoop_jobtracker, 'hadoop/jobtracker_snitch.py'): self.update_state('cluster', CluserState.BROKEN) return if not self.monitor_instance(cfg.hadoop_jobtracker, InstanceState.SNITCH_READY): self.update_state('cluster', CluserState.BROKEN) return with self.cv: while self.instances[ cfg.hadoop_namenode] != InstanceState.HADOOP_READY: self.cv.wait() util.checked_do(cfg.hadoop_jobtracker, '/start', {}) self.update_state(cfg.hadoop_jobtracker, InstanceState.HADOOP_READY) # Fork off and start our Java Hadoop monitor util.bg_exec( ['java', '-cp', 'hadoop-tools.jar', 'com.google.HadoopMonitor'], '/home/hadoop/monitor_log') with self.cv: self.cv.notifyAll()
def launch_jt(self): """Create and monitor the instance running the Jobtracker. This also blocks and waits for the NameNode, then starts the JobTracker and starts the agent that monitors Hadoop. """ # Keep this on the spawn_scheduler because it's hi-pri if not self.spawn_instance(cfg.hadoop_jobtracker, 'hadoop/jobtracker_snitch.py'): self.update_state('cluster', CluserState.BROKEN) return if not self.monitor_instance(cfg.hadoop_jobtracker, InstanceState.SNITCH_READY): self.update_state('cluster', CluserState.BROKEN) return with self.cv: while self.instances[cfg.hadoop_namenode] != InstanceState.HADOOP_READY: self.cv.wait() util.checked_do(cfg.hadoop_jobtracker, '/start', {}) self.update_state(cfg.hadoop_jobtracker, InstanceState.HADOOP_READY) # Fork off and start our Java Hadoop monitor util.bg_exec( ['java', '-cp', 'hadoop-tools.jar', 'com.google.HadoopMonitor'], '/home/hadoop/monitor_log' ) with self.cv: self.cv.notifyAll()
def start_slave(self, name): assert self.masters_up() util.checked_do(name, '/start', {}) with self.cv: self.update_state(name, InstanceState.HADOOP_READY) self.live_slaves += 1 if self.live_slaves >= cfg.needed_slaves: self.update_state('cluster', CluserState.READY)
def submit_job(self, jar, job_args): if self.state == CluserState.READY: util.checked_do(cfg.hadoop_jobtracker, '/job/start', { 'jar': jar, 'args': json.dumps(job_args) }) return True else: return False
def transfer(self, src, dst): # returns None if there's a problem, otherwise the operation name to poll if self.state != CluserState.READY: return None op = self.new_op() self.operations[op]['src'] = src self.operations[op]['dst'] = dst util.checked_do(cfg.hadoop_namenode, '/transfer', {'src': src, 'dst': dst, 'operation': op}) return self.operations[op]
def transfer(self, src, dst): # returns None if there's a problem, otherwise the operation name to poll if self.state != CluserState.READY: return None op = self.new_op() self.operations[op]['src'] = src self.operations[op]['dst'] = dst util.checked_do(cfg.hadoop_namenode, '/transfer', { 'src': src, 'dst': dst, 'operation': op }) return self.operations[op]
def send_coordinator(cmd, data, verify=False): data['secret'] = cfg.secret if verify: return util.checked_do(cfg.coordinator, cmd, data=data) else: return util.talk_to_agent(util.name_to_ip(cfg.coordinator), cmd, data=data)
def clean_hdfs(self, path): """Recursively deletes files from a HDFS path.""" util.checked_do(cfg.hadoop_namenode, '/clean', {'path': path}) return True