def parse_scheduling_class_option(scheduling_classes, num_threads): """Parse the command-line option for scheduling classes, which are formatted as: CLASS1,N1;CLASS2,N2;...""" # By default, we use the number of threads (-n option (default=1)). if scheduling_classes is None: scheduling_classes = '*,%d' % num_threads return {'*': num_threads} ret = {} class_strings = scheduling_classes.split(';') for class_string in class_strings: scheduling_class, capacity_string = class_string.split(',') capacity = int(capacity_string) ret[scheduling_class] = capacity return ret def worker_main(options): local_port = cherrypy.config.get('server.socket_port') assert (local_port) w = Worker(ciel.engine, local_port, options) w.start_running() if __name__ == '__main__': skywriting.main("worker")
run_id = args[1] if len(args) > 1 else 'allinone' base_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-')) ciel.log('Writing block store files to %s' % base_dir, 'ALLINONE', logging.INFO) if options.blockstore is not None: base_dir = options.blockstore else: base_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-')) options.blockstore = base_dir block_store = BlockStore(ciel.engine, 'localhost', 8000, base_dir, True) initial_task_descriptor, cont_ref = build_initial_task_descriptor(script_filename, block_store, 'root', 'root_cont', 'root_output') initial_task_object = build_taskpool_task_from_descriptor(initial_task_descriptor, None) task_runner = TaskRunner(initial_task_object, cont_ref, block_store, options) try: print run_id, 'SUBMITTED_JOB', now_as_timestamp() result = task_runner.run() print run_id, 'GOT_RESULT', now_as_timestamp() print block_store.retrieve_object_for_ref(result, 'json') except: pass if __name__ == '__main__': skywriting.main("allinone")
if options.workerlist is not None: master_details = {'netloc': master_netloc} master_details_as_json = simplejson.dumps(master_details) with (open(options.workerlist, "r")) as f: for worker_url in f.readlines(): try: post_string(urllib2.urlparse.urljoin(worker_url, 'control/master/'), master_details_as_json) # Worker will be created by a callback. except: ciel.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING) ciel.engine.block() # sch = SchedulerProxy(ciel.engine) # sch.subscribe() # # reaper = WorkerReaper(ciel.engine) # reaper.subscribe() # # wr = WorkflowRunner(ciel.engine) # wr.subscribe() # # te = TaskExecutor(ciel.engine) # te.subscribe() # # ph = PingHandler(ciel.engine) # ph.subscribe() if __name__ == '__main__': skywriting.main("master")
def get_log_entries(self, start_index, end_index): with self.log_lock: return self.event_log[start_index:end_index] def await_log_entries_after(self, index): with self.log_lock: while len(self.event_log) <= int(index): if self.stopping == True: break self.log_condition.wait() if self.stopping: raise Exception("Worker stopping") def worker_main(options): local_hostname = None if options.hostname is not None: local_hostname = options.hostname else: local_hostname = socket.getfqdn() local_port = cherrypy.config.get('server.socket_port') assert (local_port) w = Worker(cherrypy.engine, local_hostname, local_port, options) w.start_running() if __name__ == '__main__': skywriting.main("worker")