def __dealwith_GenerateDiskFileListTask_message(self, message, task): if message.get_isSuccess() == True: log.info(task.to_string()+' finished successfully') else: log.info(task.to_string()+' failed') message = MasterMessage(configoperations.get_master_ip(), self.__job['job-ID'], masterport, False) send_message_to_mastermanager(message)
def monitor_thread(): while True: sleep(10) try: s = xmlrpclib.ServerProxy('http://'+configoperations.get_master_ip()+':'+str(masterport)) s.check_workermanager() except Exception,e: log.warning(e)
def show_s3transfer_status(ip='127.0.0.1'): if get_local_s3transfer_status(ip) == status.S3TRANSFER.ok: if ip == configoperations.get_master_ip(): print '*' + ip + ', s3transfer is ok' else: print ip + ', s3transfer is ok' show_master_worker_num(ip) else: print 's3transfer is closed'
def __get_ready_worker(self): try: while True: while self.__doesHaveToken == False: sleep(1) machinesIpList = configoperations.read_machines_ip() if not machinesIpList: if self.__get_workerstatus(configoperations.get_master_ip()) == status.WORKER.ready: return configoperations.get_master_ip() for workerip in machinesIpList: if self.__workerSpeed.has_key(workerip) == False: self.__workerSpeed[workerip] = 0 self.__workerPriorityqueue.append(workerip) for workerip in self.__workerPriorityqueue: if self.__get_workerstatus(workerip) == status.WORKER.ready: return workerip except Exception, e: log.info(e)
def send_message_to_mastermanager(message): #when send message failed, then loop while True: try: s = xmlrpclib.ServerProxy('http://' + configoperations.get_master_ip() + ':' + str(port.ManagerPort.MasterManager), allow_none=True) s.send_message(message) log.info('send message to mastermanager successfully') return True except Exception, e: log.warning(e) log.warning('send message to mastermanager failed, will retry') sleep(10)
def __do_job_thread(self, job): try: self.__masterLock.acquire() message = MasterMessage(configoperations.get_master_ip(), job['job-ID'],masterport,True,None) self.masterstatus = status.MASTER.busy self.__bigGranularityLock.acquire() self.__workermanagerdict = dict() self.__bigGranularityLock.release() self.__job = job self.desClient = s3client.S3Client(self.__job['des-accesskey'],self.__job['des-secretkey'],self.__job['des-endpoint']) self.__build_synctask_context() self.currentJobType = self.__job['job-type'] self.taskPool = taskpool.TaskPool(self, str(configoperations.get_master_ip())+':'+str(masterport)) self.__do_synctask() send_message_to_mastermanager(message) log.info('job:' + self.__job['job-ID'] + ' finished successfully') except (ossexception.GetObjectFailed,ossexception.PutObjectFailed, jobexception.GenerateSynctaskFailed,Exception), e: log.warning(e) message.set_isSuccess(False) send_message_to_mastermanager(message) log.info('job:' + self.__job['job-ID'] + ' failed')