class JobsPool(object):
SLEEP_TIME = 300
def __init__(self, max_jobs):
self.remaining_jobs = Queue()
for i in range(max_jobs):
self.remaining_jobs.put(None)
self.jobs_lock = threading.Lock()
self.jobs = []
self._cancel = False
self._stop = False
self._check_thread = threading.Thread(target=self._check_jobs)
self._check_thread.start()
def submit(self, subfile, debug=False):
if self._stop:
raise ValueError("Pool is canceled, no more jobs can be added")
self.remaining_jobs.get(block=True, timeout=1e100)
with self.jobs_lock:
if self._stop:
self.remaining_jobs.release()
raise ValueError('Pool is canceled, no more jobs can be added')
job = Job(subfile, debug=debug)
job.start()
self.jobs.append(job)
def _check_jobs(self):
while True:
with self.jobs_lock:
for job in list(self.jobs):
if self._cancel:
if not job.is_done():
try:
job.cancel()
except BaseException:
log.exception(
'Error while trying to cancel job %d',
job.job_id)
if self._cancel or job.is_done():
self.jobs.remove(job)
self.remaining_jobs.put(None, block=False)
if not self.jobs and self._stop:
return
time.sleep(self.SLEEP_TIME)
def cancel(self):
with self.jobs_lock:
self._cancel = True
self._stop = True
def close(self):
print "close"
self._stop = True
def join(self, timeout=1e100):
self.close()
self._check_thread.join(timeout)
class JobsPool(object):
SLEEP_TIME = 300
def __init__(self, max_jobs):
self.remaining_jobs = Queue()
for i in range(max_jobs):
self.remaining_jobs.put(None)
self.jobs_lock = threading.Lock()
self.jobs = []
self._cancel = False
self._stop = False
self._check_thread = threading.Thread(target=self._check_jobs)
self._check_thread.start()
def submit(self, subfile, debug = False):
if self._stop:
raise ValueError("Pool is canceled, no more jobs can be added")
self.remaining_jobs.get(block=True, timeout=1e100)
with self.jobs_lock:
if self._stop:
self.remaining_jobs.release()
raise ValueError('Pool is canceled, no more jobs can be added')
job = Job(subfile, debug = debug)
job.start()
self.jobs.append(job)
def _check_jobs(self):
while True:
with self.jobs_lock:
for job in list(self.jobs):
if self._cancel:
if not job.is_done():
try:
job.cancel()
except BaseException:
log.exception('Error while trying to cancel job %d', job.job_id)
if self._cancel or job.is_done():
self.jobs.remove(job)
self.remaining_jobs.put(None, block=False)
if not self.jobs and self._stop:
return
time.sleep(self.SLEEP_TIME)
def cancel(self):
with self.jobs_lock:
self._cancel = True
self._stop = True
def close(self):
print "close"
self._stop = True
def join(self, timeout=1e100):
self.close()
self._check_thread.join(timeout)