def start(self):
utils.info('Starting Master')
self.main_thread = MainThread(self.db, self)
self.main_thread.start()
if self.enable_ping:
self.ping_thread.start()
self.ping_thread = PingThread(self.db)
def process_jobs(self):
jobs = self.db.jobs
for job in jobs.find():
if job['status'] == 'active':
if self.job_is_completed(job):
utils.info('Job completed: {0}'.format(job['name']))
jobs.update({'_id' : job['_id']}, {'$set' : {'status' : 'completed'}})
def process_commands(self):
for command in self.db.master_commands.find():
if command['command'] in self.commands:
utils.info('Got command: {0}'.format(command['command']))
self.commands[command['command']](command['params'])
else:
utils.error('Command not recognised: {0} - deleting'.format(command['command']))
self.db.master_commands.remove({'_id': command['_id']})
def scan_jobs(self):
try:
job_dirs = os.listdir(self.controller.job_dir)
except:
utils.error('Directory error: {0}'.format(self.controller.job_dir))
return
for job_dir in job_dirs:
abs_job_dir = os.path.join(self.controller.job_dir, job_dir)
job_info_file = os.path.join(abs_job_dir, 'job_info.json')
if os.path.exists(job_info_file):
utils.info('Found job info file: {0}'.format(job_info_file))
job_info = None
try:
job_info = json.loads(open(job_info_file).read())
except:
utils.error('Error parsing .json file')
return
finally:
os.rename(job_info_file, '{0}.backup'.format(job_info_file))
job_id = self.db.jobs.insert(
{'name' : job_info['name'],
'root' : abs_job_dir,
'module' : job_info['module'],
'status' : 'active',
'params' : job_info['module_data'],
'attempts' : 0,
'fragments' : None,
'commands' : []
})
utils.info('Adding job: {0}'.format(job_info['name']))
fragment_ids = self.create_fragments(job_id, job_info['module'])
if fragment_ids is not None:
self.db.jobs.update({'_id' : job_id}, {'$set' : {'fragments' : fragment_ids}})
else:
utils.error('Failed to creat fragments for {0}'.format(job_info['name']))
self.db.jobs.update({'_id' : job_id}, {'$set' : {'status' : 'failed'}})
def process_slaves(self):
slaves = self.db.slaves
for slave in self.db.slaves.find():
if slave['status'] == 'waiting':
fragments = self.db.fragments
for fragment in fragments.find():
if fragment['status'] == 'waiting':
jobs = self.db.jobs
job = jobs.find_one({'_id' : fragment['job']})
if job is not None:
if job['status'] == 'active':
utils.info('Submitting fragment {0} to slave {1}'.format(fragment['name'], slave['name']))
fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'assigned', 'owner' : slave['_id'], 'attempts' : fragment['attempts'] + 1}})
slaves.update({'_id' : slave['_id']}, {'$set' : {'fragment' : fragment['_id'], 'status' : 'busy'}})
return
elif slave['status'] == 'unreachable':
for module in self.controller.slave_modules:
if module.Controller.get_type() == slave['type']:
utils.warning('Restarting slave: {0}'.format(slave['name']))
module.restart_slave(slave['_id'])
if slave['attempts'] < self.controller.fragment_attempt_limit:
utils.info('Releasing fragment {0}'.format(fragment['name']))
fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'waiting', 'ownder' : None, 'attempts' : fragment['attempts'] + 1}})
else:
utils.info('Cancelling fragment {0} after {1} attempts'.format(fragment['name'], fragment['attempts']))
fragments.update({'_id' : fragment['_id']}, {'$set' : {'status' : 'cancelled', 'ownder' : None}})
slave.update({'id' : slave['_id']}, {'$set' : {'fragment' : None, 'status' : 'restarting'}})
return
def slave_create(db, data):
db.master_commands.insert({'command' : 'slave_create', 'params' : {}})
utils.info('Command added')
