def delete_runner(self, qdel_cmd):
if self.jobid:
print run_bash(qdel_cmd+" "+self.jobid,10,3) ## means 10 tries with 3 seconds
if self.inputfolder:
print "remove "+self.inputfolder
shutil.rmtree(self.inputfolder, ignore_errors=True)
if self.submitscript:
print "delete "+self.submitscript
os.remove(self.submitscript)
def activate_runners(self):
qsub_cmd = self.config.give_value_from_option("qsub_cmd","runner_option","qsub -j y -o /dev/null")
qsub_to_add=os.getenv("QSUB_ADD")
if qsub_to_add:
qsub_cmd+=" "+qsub_to_add
for akt_runner in self.runners:
if akt_runner.state == "created":
ret_string = run_bash(qsub_cmd+" "+akt_runner.submitscript,1,5)
try:
print qsub_cmd+" "+akt_runner.submitscript
print ret_string
number = ret_string.split()[2]
int(number)
akt_runner.jobid = number
akt_runner.state = "submitted"
except:
print "Not sure if submit was succesfull"
akt_runner.state = "unknown"
def check_runner_qstat_naf(self):
qstatxml_cmd = self.config.give_value_from_option("qstat_xml","runner_option","qstat -xml")
xml_string = run_bash(qstatxml_cmd,5,1,2) ## means 10 tries with time: 1s, (1+2)s, ...
# xml_string = run_bash(qstatxml_cmd,2,1,2) ## means 10 tries with time: 1s, (1+2)s, ...
if xml_string.strip():
## First save the file, from time to time qstat is hanging, and to use the old qstat
self.qstattimestamp = time.time()
save_xml=open(self.folder.rstrip('/')+'/save_qstat.xml','w')
save_xml.write(xml_string)
save_xml.close()
if os.path.exists(self.folder.rstrip('/')+'/save_qstat.xml'):
#self.qstat = ET.fromstring(xml_string)
self.qstat = ET.parse(self.folder.rstrip('/')+'/save_qstat.xml').getroot()
for child in self.qstat.iter("job_list"):
job_state = child.attrib.get('state').strip()
if child.find('state').text.strip() == 'Eqw':
job_state = "error"
job_name = child.find('JB_name').text.split('.')[0].strip()
job_id = child.find('JB_job_number').text.strip()
if not self.match_job_to_runner(job_name,job_state,job_id):
print "Can't match the job with id: "+job_id+" to anny runner"
self.delete_runners_with_state(["recovered","error"]) ## if job is not matched at and job state is error delet this runner
else:
print "can't update runner state. (qstat is haning) and no old qtat config is available"
def list_runner_folder(self):
if os.path.isdir(self.inputfolder):
run_bash("touch "+self.inputfolder) ## need to get the an updated file contend
return os.listdir(self.inputfolder)
return []
