class JobWorker(threading.Thread): def __init__(self, cb, sensor_id, result_queue): super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue def run(self): try: self.lr_session = self.cb.live_response.request_session(self.sensor_id) self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) while True: work_item = self.job_queue.get(block=True) if not work_item: self.job_queue.task_done() return self.run_job(work_item) self.result_queue.put(CompletionNotification(self.sensor_id)) self.job_queue.task_done() except Exception as e: self.result_queue.put(WorkerStatus(self.sensor_id, status="error", exception=e)) finally: if self.lr_session: self.lr_session.close() self.result_queue.put(WorkerStatus(self.sensor_id, status="exiting")) def run_job(self, work_item): try: work_item.future.set_result(work_item.fn(self.lr_session)) except Exception as e: work_item.future.set_exception(e)
def __init__(self, cb, sensor_id, result_queue): super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue
def __init__(self, cb, max_workers=10): super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} self._idle_workers = set() self._unscheduled_jobs = defaultdict(list) self._max_workers = max_workers self.schedule_queue = Queue()
class JobWorker(threading.Thread): """Thread object that executes individual Live Response jobs.""" def __init__(self, cb, sensor_id, result_queue): """ Initialize the JobWorker. Args: cb (BaseAPI): The CBAPI object reference. sensor_id (int): The ID of the sensor being used. result_queue (Queue): The queue where results are placed. """ super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue def run(self): """Execute the job worker.""" try: self.lr_session = self.cb.live_response.request_session( self.sensor_id) self.result_queue.put(WorkerStatus(self.sensor_id, status="ready")) while True: work_item = self.job_queue.get(block=True) if not work_item: self.job_queue.task_done() return self.run_job(work_item) self.result_queue.put(CompletionNotification(self.sensor_id)) self.job_queue.task_done() except Exception as e: self.result_queue.put( WorkerStatus(self.sensor_id, status="error", exception=e)) finally: if self.lr_session: self.lr_session.close() self.result_queue.put( WorkerStatus(self.sensor_id, status="exiting")) def run_job(self, work_item): """ Execute an individual WorkItem. Args: work_item (WorkItem): The work item to execute. """ try: work_item.future.set_result(work_item.fn(self.lr_session)) except Exception as e: work_item.future.set_exception(e)
def __init__(self, cb, max_workers=10): """ Initialize the LiveResponseJobScheduler. Args: cb (BaseAPI): The CBAPI object reference. max_workers (int): Maximum number of JobWorker threads to use. """ super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} self._idle_workers = set() self._unscheduled_jobs = defaultdict(list) self._max_workers = max_workers self.schedule_queue = Queue()
def __init__(self, cb, sensor_id, result_queue): """ Initialize the JobWorker. Args: cb (BaseAPI): The CBAPI object reference. sensor_id (int): The ID of the sensor being used. result_queue (Queue): The queue where results are placed. """ super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue
def __init__(self, cb, sensor_id, result_queue): super(JobWorker, self).__init__() self.cb = cb self.sensor_id = sensor_id self.job_queue = Queue() self.lr_session = None self.result_queue = result_queue
def __init__(self, cb, max_workers=10): super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} self._idle_workers = set() self._unscheduled_jobs = defaultdict(list) self._max_workers = max_workers self.schedule_queue = Queue()
class LiveResponseJobScheduler(threading.Thread): daemon = True def __init__(self, cb, max_workers=10): super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} self._idle_workers = set() self._unscheduled_jobs = defaultdict(list) self._max_workers = max_workers self.schedule_queue = Queue() def run(self): log.debug("Starting Live Response Job Scheduler") while True: log.debug("Waiting for item on Scheduler Queue") item = self.schedule_queue.get(block=True) log.debug("Got item: {0}".format(item)) if isinstance(item, WorkItem): # new WorkItem available self._unscheduled_jobs[item.sensor_id].append(item) elif isinstance(item, CompletionNotification): # job completed self._idle_workers.add(item.sensor_id) elif isinstance(item, WorkerStatus): if item.status == "error": log.error( "Error encountered by JobWorker[{0}]: {1}".format( item.sensor_id, item.exception)) elif item.status == "exiting": log.debug("JobWorker[{0}] has exited, waiting...".format( item.sensor_id)) self._job_workers[item.sensor_id].join() log.debug("JobWorker[{0}] deleted".format(item.sensor_id)) del self._job_workers[item.sensor_id] try: self._idle_workers.remove(item.sensor_id) except KeyError: pass elif item.status == "ready": log.debug( "JobWorker[{0}] now ready to accept jobs, session established" .format(item.sensor_id)) self._idle_workers.add(item.sensor_id) else: log.debug("Unknown status from JobWorker[{0}]: {1}".format( item.sensor_id, item.status)) else: log.debug( "Received unknown item on the scheduler Queue, exiting") # exiting the scheduler if we get None # TODO: wait for all worker threads to exit return self._schedule_jobs() def _schedule_jobs(self): log.debug("Entering scheduler") # First, see if there are new jobs to schedule on idle workers. self._schedule_existing_workers() # If we have jobs scheduled to run on sensors with no current associated worker, let's spawn new ones. if set(self._unscheduled_jobs.keys()) - self._idle_workers: self._cleanup_idle_workers() self._spawn_new_workers() self._schedule_existing_workers() def _cleanup_idle_workers(self, max=None): if not max: max = self._max_workers for sensor in list(self._idle_workers)[:max]: log.debug("asking worker for sensor id {0} to exit".format(sensor)) self._job_workers[sensor].job_queue.put(None) def _schedule_existing_workers(self): log.debug("There are idle workers for sensor ids {0}".format( self._idle_workers)) intersection = self._idle_workers.intersection( set(self._unscheduled_jobs.keys())) log.debug( "{0} jobs ready to execute in existing execution slots".format( len(intersection))) for sensor in intersection: item = self._unscheduled_jobs[sensor].pop(0) self._job_workers[sensor].job_queue.put(item) self._idle_workers.remove(item.sensor_id) self._cleanup_unscheduled_jobs() def _cleanup_unscheduled_jobs(self): marked_for_deletion = [] for k in self._unscheduled_jobs.keys(): if len(self._unscheduled_jobs[k]) == 0: marked_for_deletion.append(k) for k in marked_for_deletion: del self._unscheduled_jobs[k] def submit_job(self, work_item): self.schedule_queue.put(work_item) def _spawn_new_workers(self): if len(self._job_workers) >= self._max_workers: return schedule_max = self._max_workers - len(self._job_workers) sensors = [ s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs and s.id not in self._job_workers and s.status == "Online" ] sensors_to_schedule = sorted( sensors, key=lambda x: x.next_checkin_time)[:schedule_max] log.debug("Spawning new workers to handle these sensors: {0}".format( sensors_to_schedule)) for sensor in sensors_to_schedule: log.debug("Spawning new JobWorker for sensor id {0}".format( sensor.id)) self._job_workers[sensor.id] = JobWorker(self._cb, sensor.id, self.schedule_queue) self._job_workers[sensor.id].start()
#!/usr/bin/env python from cbapi.response.models import Binary from cbapi.example_helpers import build_cli_parser, get_cb_response_object import sys from cbapi.six.moves.queue import Queue import os import threading import json import logging worker_queue = Queue(maxsize=50) def get_path_for_md5(d, basepath=''): d = d.upper() return os.path.join(basepath, d[:3], d[3:6], d) def create_directory(pathname): try: os.makedirs(os.path.dirname(pathname)) except: pass class BinaryWorker(threading.Thread): def __init__(self, basepath): self.basepath = basepath threading.Thread.__init__(self)
class LiveResponseJobScheduler(threading.Thread): daemon = True def __init__(self, cb, max_workers=10): super(LiveResponseJobScheduler, self).__init__() self._cb = cb self._job_workers = {} self._idle_workers = set() self._unscheduled_jobs = defaultdict(list) self._max_workers = max_workers self.schedule_queue = Queue() def run(self): log.debug("Starting Live Response Job Scheduler") while True: log.debug("Waiting for item on Scheduler Queue") item = self.schedule_queue.get(block=True) log.debug("Got item: {0}".format(item)) if isinstance(item, WorkItem): # new WorkItem available self._unscheduled_jobs[item.sensor_id].append(item) elif isinstance(item, CompletionNotification): # job completed self._idle_workers.add(item.sensor_id) elif isinstance(item, WorkerStatus): if item.status == "error": log.error("Error encountered by JobWorker[{0}]: {1}".format(item.sensor_id, item.exception)) elif item.status == "exiting": log.debug("JobWorker[{0}] has exited, waiting...".format(item.sensor_id)) self._job_workers[item.sensor_id].join() log.debug("JobWorker[{0}] deleted".format(item.sensor_id)) del self._job_workers[item.sensor_id] try: self._idle_workers.remove(item.sensor_id) except KeyError: pass elif item.status == "ready": log.debug("JobWorker[{0}] now ready to accept jobs, session established".format(item.sensor_id)) self._idle_workers.add(item.sensor_id) else: log.debug("Unknown status from JobWorker[{0}]: {1}".format(item.sensor_id, item.status)) else: log.debug("Received unknown item on the scheduler Queue, exiting") # exiting the scheduler if we get None # TODO: wait for all worker threads to exit return self._schedule_jobs() def _schedule_jobs(self): log.debug("Entering scheduler") # First, see if there are new jobs to schedule on idle workers. self._schedule_existing_workers() # If we have jobs scheduled to run on sensors with no current associated worker, let's spawn new ones. if set(self._unscheduled_jobs.keys()) - self._idle_workers: self._cleanup_idle_workers() self._spawn_new_workers() self._schedule_existing_workers() def _cleanup_idle_workers(self, max=None): if not max: max = self._max_workers for sensor in list(self._idle_workers)[:max]: log.debug("asking worker for sensor id {0} to exit".format(sensor)) self._job_workers[sensor].job_queue.put(None) def _schedule_existing_workers(self): log.debug("There are idle workers for sensor ids {0}".format(self._idle_workers)) intersection = self._idle_workers.intersection(set(self._unscheduled_jobs.keys())) log.debug("{0} jobs ready to execute in existing execution slots".format(len(intersection))) for sensor in intersection: item = self._unscheduled_jobs[sensor].pop(0) self._job_workers[sensor].job_queue.put(item) self._idle_workers.remove(item.sensor_id) self._cleanup_unscheduled_jobs() def _cleanup_unscheduled_jobs(self): marked_for_deletion = [] for k in self._unscheduled_jobs.keys(): if len(self._unscheduled_jobs[k]) == 0: marked_for_deletion.append(k) for k in marked_for_deletion: del self._unscheduled_jobs[k] def submit_job(self, work_item): self.schedule_queue.put(work_item) def _spawn_new_workers(self): if len(self._job_workers) >= self._max_workers: return schedule_max = self._max_workers - len(self._job_workers) sensors = [s for s in self._cb.select(Sensor) if s.id in self._unscheduled_jobs and s.id not in self._job_workers and s.status == "Online"] sensors_to_schedule = sorted(sensors, key=lambda x: x.next_checkin_time)[:schedule_max] log.debug("Spawning new workers to handle these sensors: {0}".format(sensors_to_schedule)) for sensor in sensors_to_schedule: log.debug("Spawning new JobWorker for sensor id {0}".format(sensor.id)) self._job_workers[sensor.id] = JobWorker(self._cb, sensor.id, self.schedule_queue) self._job_workers[sensor.id].start()