class JobsPool(object): SLEEP_TIME = 300 def __init__(self, max_jobs): self.remaining_jobs = Queue() for i in range(max_jobs): self.remaining_jobs.put(None) self.jobs_lock = threading.Lock() self.jobs = [] self._cancel = False self._stop = False self._check_thread = threading.Thread(target=self._check_jobs) self._check_thread.start() def submit(self, subfile, debug=False): if self._stop: raise ValueError("Pool is canceled, no more jobs can be added") self.remaining_jobs.get(block=True, timeout=1e100) with self.jobs_lock: if self._stop: self.remaining_jobs.release() raise ValueError('Pool is canceled, no more jobs can be added') job = Job(subfile, debug=debug) job.start() self.jobs.append(job) def _check_jobs(self): while True: with self.jobs_lock: for job in list(self.jobs): if self._cancel: if not job.is_done(): try: job.cancel() except BaseException: log.exception( 'Error while trying to cancel job %d', job.job_id) if self._cancel or job.is_done(): self.jobs.remove(job) self.remaining_jobs.put(None, block=False) if not self.jobs and self._stop: return time.sleep(self.SLEEP_TIME) def cancel(self): with self.jobs_lock: self._cancel = True self._stop = True def close(self): print "close" self._stop = True def join(self, timeout=1e100): self.close() self._check_thread.join(timeout)
class JobsPool(object): SLEEP_TIME = 300 def __init__(self, max_jobs): self.remaining_jobs = Queue() for i in range(max_jobs): self.remaining_jobs.put(None) self.jobs_lock = threading.Lock() self.jobs = [] self._cancel = False self._stop = False self._check_thread = threading.Thread(target=self._check_jobs) self._check_thread.start() def submit(self, subfile, debug = False): if self._stop: raise ValueError("Pool is canceled, no more jobs can be added") self.remaining_jobs.get(block=True, timeout=1e100) with self.jobs_lock: if self._stop: self.remaining_jobs.release() raise ValueError('Pool is canceled, no more jobs can be added') job = Job(subfile, debug = debug) job.start() self.jobs.append(job) def _check_jobs(self): while True: with self.jobs_lock: for job in list(self.jobs): if self._cancel: if not job.is_done(): try: job.cancel() except BaseException: log.exception('Error while trying to cancel job %d', job.job_id) if self._cancel or job.is_done(): self.jobs.remove(job) self.remaining_jobs.put(None, block=False) if not self.jobs and self._stop: return time.sleep(self.SLEEP_TIME) def cancel(self): with self.jobs_lock: self._cancel = True self._stop = True def close(self): print "close" self._stop = True def join(self, timeout=1e100): self.close() self._check_thread.join(timeout)