def start(self): utils.info('Starting Master') self.main_thread = MainThread(self.db, self) self.main_thread.start() if self.enable_ping: self.ping_thread.start() self.ping_thread = PingThread(self.db)
def process_jobs(self): jobs = self.db.jobs for job in jobs.find(): if job['status'] == 'active': if self.job_is_completed(job): utils.info('Job completed: {0}'.format(job['name'])) jobs.update({'_id' : job['_id']}, {'$set' : {'status' : 'completed'}})
def process_commands(self): for command in self.db.master_commands.find(): if command['command'] in self.commands: utils.info('Got command: {0}'.format(command['command'])) self.commands[command['command']](command['params']) else: utils.error('Command not recognised: {0} - deleting'.format(command['command'])) self.db.master_commands.remove({'_id': command['_id']})
def scan_jobs(self): try: job_dirs = os.listdir(self.controller.job_dir) except: utils.error('Directory error: {0}'.format(self.controller.job_dir)) return for job_dir in job_dirs: abs_job_dir = os.path.join(self.controller.job_dir, job_dir) job_info_file = os.path.join(abs_job_dir, 'job_info.json') if os.path.exists(job_info_file): utils.info('Found job info file: {0}'.format(job_info_file)) job_info = None try: job_info = json.loads(open(job_info_file).read()) except: utils.error('Error parsing .json file') return finally: os.rename(job_info_file, '{0}.backup'.format(job_info_file)) job_id = self.db.jobs.insert( {'name' : job_info['name'], 'root' : abs_job_dir, 'module' : job_info['module'], 'status' : 'active', 'params' : job_info['module_data'], 'attempts' : 0, 'fragments' : None, 'commands' : [] }) utils.info('Adding job: {0}'.format(job_info['name'])) fragment_ids = self.create_fragments(job_id, job_info['module']) if fragment_ids is not None: self.db.jobs.update({'_id' : job_id}, {'$set' : {'fragments' : fragment_ids}}) else: utils.error('Failed to creat fragments for {0}'.format(job_info['name'])) self.db.jobs.update({'_id' : job_id}, {'$set' : {'status' : 'failed'}})
def process_slaves(self): slaves = self.db.slaves for slave in self.db.slaves.find(): if slave['status'] == 'waiting': fragments = self.db.fragments for fragment in fragments.find(): if fragment['status'] == 'waiting': jobs = self.db.jobs job = jobs.find_one({'_id' : fragment['job']}) if job is not None: if job['status'] == 'active': utils.info('Submitting fragment {0} to slave {1}'.format(fragment['name'], slave['name'])) fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'assigned', 'owner' : slave['_id'], 'attempts' : fragment['attempts'] + 1}}) slaves.update({'_id' : slave['_id']}, {'$set' : {'fragment' : fragment['_id'], 'status' : 'busy'}}) return elif slave['status'] == 'unreachable': for module in self.controller.slave_modules: if module.Controller.get_type() == slave['type']: utils.warning('Restarting slave: {0}'.format(slave['name'])) module.restart_slave(slave['_id']) if slave['attempts'] < self.controller.fragment_attempt_limit: utils.info('Releasing fragment {0}'.format(fragment['name'])) fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'waiting', 'ownder' : None, 'attempts' : fragment['attempts'] + 1}}) else: utils.info('Cancelling fragment {0} after {1} attempts'.format(fragment['name'], fragment['attempts'])) fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'cancelled', 'ownder' : None}}) slave.update({'id' : slave['_id']}, {'$set' : {'fragment' : None, 'status' : 'restarting'}}) return
def slave_create(db, data): db.master_commands.insert({'command' : 'slave_create', 'params' : {}}) utils.info('Command added')