def __init__(self, bus, hostname, port, options): plugins.SimplePlugin.__init__(self, bus) self.id = None self.hostname = hostname self.port = port self.master_url = options.master self.master_proxy = MasterProxy(self, bus, self.master_url) self.master_proxy.subscribe() if options.blockstore is None: block_store_dir = tempfile.mkdtemp( prefix=os.getenv('TEMP', default='/tmp/sw-files-')) else: block_store_dir = options.blockstore self.block_store = BlockStore(cherrypy.engine, self.hostname, self.port, block_store_dir, ignore_blocks=options.ignore_blocks) self.block_store.build_pin_set() self.upload_deferred_work = DeferredWorkPlugin(bus, 'upload_work') self.upload_deferred_work.subscribe() self.upload_manager = UploadManager(self.block_store, self.upload_deferred_work) self.execution_features = ExecutionFeatures() self.task_executor = TaskExecutorPlugin(bus, self.block_store, self.master_proxy, self.execution_features, 1) self.task_executor.subscribe() self.server_root = WorkerRoot(self) self.pinger = Pinger(bus, self.master_proxy, None, 30) self.pinger.subscribe() self.stopping = False self.event_log = [] self.log_lock = Lock() self.log_condition = Condition(self.log_lock) self.cherrypy_conf = {} cherrypy.config.update({"server.thread_pool": 20}) if options.staticbase is not None: self.cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase } if options.lib is not None: self.cherrypy_conf["/stdlib"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.lib } self.subscribe()
def worker_process_main(base_dir, task_queue, response_queue): master_proxy = QueueMasterProxy(response_queue) execution_features = ExecutionFeatures() block_store = BlockStore(ciel.engine, 'localhost', 8000, base_dir, True) # XXX: Broken because we now need a pseudoworker in place of a block_store. thread_task_executor = TaskExecutorPlugin(ciel.engine, PseudoWorker(block_store), master_proxy, execution_features, 1) while True: task = task_queue.get() if isinstance(task, ThreadTerminator): return task_descriptor = task.as_descriptor(False) thread_task_executor.handle_input(task_descriptor)
def worker_process_main(base_dir, task_queue, response_queue): master_proxy = QueueMasterProxy(response_queue) execution_features = ExecutionFeatures() block_store = BlockStore(ciel.engine, 'localhost', 8000, base_dir, True) # XXX: Broken because we now need a pseudoworker in place of a block_store. thread_task_executor = TaskExecutorPlugin(ciel.engine, PseudoWorker(block_store), master_proxy, execution_features, 1) while True: task = task_queue.get() if isinstance(task, ThreadTerminator): return task_descriptor = task.as_descriptor(False) thread_task_executor.handle_input(task_descriptor)
def __init__(self, bus, hostname, port, options): plugins.SimplePlugin.__init__(self, bus) self.id = None self.hostname = hostname self.port = port self.master_url = options.master self.master_proxy = MasterProxy(self, bus, self.master_url) self.master_proxy.subscribe() if options.blockstore is None: block_store_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-')) else: block_store_dir = options.blockstore self.block_store = BlockStore(cherrypy.engine, self.hostname, self.port, block_store_dir, ignore_blocks=options.ignore_blocks) self.block_store.build_pin_set() self.upload_deferred_work = DeferredWorkPlugin(bus, 'upload_work') self.upload_deferred_work.subscribe() self.upload_manager = UploadManager(self.block_store, self.upload_deferred_work) self.execution_features = ExecutionFeatures() self.task_executor = TaskExecutorPlugin(bus, self.block_store, self.master_proxy, self.execution_features, 1) self.task_executor.subscribe() self.server_root = WorkerRoot(self) self.pinger = Pinger(bus, self.master_proxy, None, 30) self.pinger.subscribe() self.stopping = False self.event_log = [] self.log_lock = Lock() self.log_condition = Condition(self.log_lock) self.cherrypy_conf = {} if options.staticbase is not None: self.cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase } if options.lib is not None: self.cherrypy_conf["/stdlib"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.lib } self.subscribe()
class Worker(plugins.SimplePlugin): def __init__(self, bus, hostname, port, options): plugins.SimplePlugin.__init__(self, bus) self.id = None self.hostname = hostname self.port = port self.master_url = options.master self.master_proxy = MasterProxy(self, bus, self.master_url) self.master_proxy.subscribe() if options.blockstore is None: block_store_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-')) else: block_store_dir = options.blockstore self.block_store = BlockStore(cherrypy.engine, self.hostname, self.port, block_store_dir, ignore_blocks=options.ignore_blocks) self.block_store.build_pin_set() self.upload_deferred_work = DeferredWorkPlugin(bus, 'upload_work') self.upload_deferred_work.subscribe() self.upload_manager = UploadManager(self.block_store, self.upload_deferred_work) self.execution_features = ExecutionFeatures() self.task_executor = TaskExecutorPlugin(bus, self.block_store, self.master_proxy, self.execution_features, 1) self.task_executor.subscribe() self.server_root = WorkerRoot(self) self.pinger = Pinger(bus, self.master_proxy, None, 30) self.pinger.subscribe() self.stopping = False self.event_log = [] self.log_lock = Lock() self.log_condition = Condition(self.log_lock) self.cherrypy_conf = {} if options.staticbase is not None: self.cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase } if options.lib is not None: self.cherrypy_conf["/stdlib"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.lib } self.subscribe() def subscribe(self): self.bus.subscribe('stop', self.stop, priority=10) self.bus.subscribe("worker_event", self.add_log_entry) def unsubscribe(self): self.bus.unsubscribe('stop', self.stop) self.bus.unsubscribe("worker_event", self.add_log_entry) def netloc(self): return '%s:%d' % (self.hostname, self.port) def as_descriptor(self): return {'netloc': self.netloc(), 'features': self.execution_features.all_features(), 'has_blocks': not self.block_store.is_empty()} def set_master(self, master_details): self.master_url = master_details['master'] self.master_proxy.change_master(self.master_url) self.pinger.poke() def start_running(self): cherrypy.engine.start() cherrypy.tree.mount(self.server_root, "", self.cherrypy_conf) if hasattr(cherrypy.engine, "signal_handler"): cherrypy.engine.signal_handler.subscribe() if hasattr(cherrypy.engine, "console_control_handler"): cherrypy.engine.console_control_handler.subscribe() cherrypy.engine.block() def stop(self): with self.log_lock: self.stopping = True self.log_condition.notify_all() def submit_task(self, task_descriptor): cherrypy.engine.publish("worker_event", "Start task " + repr(task_descriptor["task_id"])) cherrypy.engine.publish('execute_task', task_descriptor) def abort_task(self, task_id): cherrypy.engine.publish("worker_event", "Abort task " + repr(task_id)) self.task_executor.abort_task(task_id) def notify_task_streams_done(self, task_id): self.task_executor.notify_streams_done(task_id) def add_log_entry(self, log_string): with self.log_lock: self.event_log.append((datetime.now(), log_string)) self.log_condition.notify_all() def get_log_entries(self, start_index, end_index): with self.log_lock: return self.event_log[start_index:end_index] def await_log_entries_after(self, index): with self.log_lock: while len(self.event_log) <= int(index): if self.stopping == True: break self.log_condition.wait() if self.stopping: raise Exception("Worker stopping")
class Worker(plugins.SimplePlugin): def __init__(self, bus, hostname, port, options): plugins.SimplePlugin.__init__(self, bus) self.id = None self.hostname = hostname self.port = port self.master_url = options.master self.master_proxy = MasterProxy(self, bus, self.master_url) self.master_proxy.subscribe() if options.blockstore is None: block_store_dir = tempfile.mkdtemp( prefix=os.getenv('TEMP', default='/tmp/sw-files-')) else: block_store_dir = options.blockstore self.block_store = BlockStore(cherrypy.engine, self.hostname, self.port, block_store_dir, ignore_blocks=options.ignore_blocks) self.block_store.build_pin_set() self.upload_deferred_work = DeferredWorkPlugin(bus, 'upload_work') self.upload_deferred_work.subscribe() self.upload_manager = UploadManager(self.block_store, self.upload_deferred_work) self.execution_features = ExecutionFeatures() self.task_executor = TaskExecutorPlugin(bus, self.block_store, self.master_proxy, self.execution_features, 1) self.task_executor.subscribe() self.server_root = WorkerRoot(self) self.pinger = Pinger(bus, self.master_proxy, None, 30) self.pinger.subscribe() self.stopping = False self.event_log = [] self.log_lock = Lock() self.log_condition = Condition(self.log_lock) self.cherrypy_conf = {} cherrypy.config.update({"server.thread_pool": 20}) if options.staticbase is not None: self.cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase } if options.lib is not None: self.cherrypy_conf["/stdlib"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.lib } self.subscribe() def subscribe(self): self.bus.subscribe('stop', self.stop, priority=10) self.bus.subscribe("worker_event", self.add_log_entry) def unsubscribe(self): self.bus.unsubscribe('stop', self.stop) self.bus.unsubscribe("worker_event", self.add_log_entry) def netloc(self): return '%s:%d' % (self.hostname, self.port) def as_descriptor(self): return { 'netloc': self.netloc(), 'features': self.execution_features.all_features(), 'has_blocks': not self.block_store.is_empty() } def set_master(self, master_details): self.master_url = master_details['master'] self.master_proxy.change_master(self.master_url) self.pinger.poke() def start_running(self): cherrypy.engine.start() cherrypy.tree.mount(self.server_root, "", self.cherrypy_conf) if hasattr(cherrypy.engine, "signal_handler"): cherrypy.engine.signal_handler.subscribe() if hasattr(cherrypy.engine, "console_control_handler"): cherrypy.engine.console_control_handler.subscribe() cherrypy.engine.block() def stop(self): with self.log_lock: self.stopping = True self.log_condition.notify_all() def submit_task(self, task_descriptor): cherrypy.engine.publish( "worker_event", "Start task " + repr(task_descriptor["task_id"])) cherrypy.engine.publish('execute_task', task_descriptor) def abort_task(self, task_id): cherrypy.engine.publish("worker_event", "Abort task " + repr(task_id)) self.task_executor.abort_task(task_id) def notify_task_streams_done(self, task_id): self.task_executor.notify_streams_done(task_id) def add_log_entry(self, log_string): with self.log_lock: self.event_log.append((datetime.now(), log_string)) self.log_condition.notify_all() def get_log_entries(self, start_index, end_index): with self.log_lock: return self.event_log[start_index:end_index] def await_log_entries_after(self, index): with self.log_lock: while len(self.event_log) <= int(index): if self.stopping == True: break self.log_condition.wait() if self.stopping: raise Exception("Worker stopping")