def parse_scheduling_class_option(scheduling_classes, num_threads):
"""Parse the command-line option for scheduling classes, which are formatted as:
CLASS1,N1;CLASS2,N2;..."""
# By default, we use the number of threads (-n option (default=1)).
if scheduling_classes is None:
scheduling_classes = '*,%d' % num_threads
return {'*': num_threads}
ret = {}
class_strings = scheduling_classes.split(';')
for class_string in class_strings:
scheduling_class, capacity_string = class_string.split(',')
capacity = int(capacity_string)
ret[scheduling_class] = capacity
return ret
def worker_main(options):
local_port = cherrypy.config.get('server.socket_port')
assert (local_port)
w = Worker(ciel.engine, local_port, options)
w.start_running()
if __name__ == '__main__':
skywriting.main("worker")
run_id = args[1] if len(args) > 1 else 'allinone'
base_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
ciel.log('Writing block store files to %s' % base_dir, 'ALLINONE', logging.INFO)
if options.blockstore is not None:
base_dir = options.blockstore
else:
base_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
options.blockstore = base_dir
block_store = BlockStore(ciel.engine, 'localhost', 8000, base_dir, True)
initial_task_descriptor, cont_ref = build_initial_task_descriptor(script_filename, block_store, 'root', 'root_cont', 'root_output')
initial_task_object = build_taskpool_task_from_descriptor(initial_task_descriptor, None)
task_runner = TaskRunner(initial_task_object, cont_ref, block_store, options)
try:
print run_id, 'SUBMITTED_JOB', now_as_timestamp()
result = task_runner.run()
print run_id, 'GOT_RESULT', now_as_timestamp()
print block_store.retrieve_object_for_ref(result, 'json')
except:
pass
if __name__ == '__main__':
skywriting.main("allinone")
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
post_string(urllib2.urlparse.urljoin(worker_url, 'control/master/'), master_details_as_json)
# Worker will be created by a callback.
except:
ciel.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING)
ciel.engine.block()
# sch = SchedulerProxy(ciel.engine)
# sch.subscribe()
#
# reaper = WorkerReaper(ciel.engine)
# reaper.subscribe()
#
# wr = WorkflowRunner(ciel.engine)
# wr.subscribe()
#
# te = TaskExecutor(ciel.engine)
# te.subscribe()
#
# ph = PingHandler(ciel.engine)
# ph.subscribe()
if __name__ == '__main__':
skywriting.main("master")
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
post_string(urllib2.urlparse.urljoin(worker_url, 'control/master/'), master_details_as_json)
# Worker will be created by a callback.
except:
ciel.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING)
ciel.engine.block()
# sch = SchedulerProxy(ciel.engine)
# sch.subscribe()
#
# reaper = WorkerReaper(ciel.engine)
# reaper.subscribe()
#
# wr = WorkflowRunner(ciel.engine)
# wr.subscribe()
#
# te = TaskExecutor(ciel.engine)
# te.subscribe()
#
# ph = PingHandler(ciel.engine)
# ph.subscribe()
if __name__ == '__main__':
skywriting.main("master")
def get_log_entries(self, start_index, end_index):
with self.log_lock:
return self.event_log[start_index:end_index]
def await_log_entries_after(self, index):
with self.log_lock:
while len(self.event_log) <= int(index):
if self.stopping == True:
break
self.log_condition.wait()
if self.stopping:
raise Exception("Worker stopping")
def worker_main(options):
local_hostname = None
if options.hostname is not None:
local_hostname = options.hostname
else:
local_hostname = socket.getfqdn()
local_port = cherrypy.config.get('server.socket_port')
assert (local_port)
w = Worker(cherrypy.engine, local_hostname, local_port, options)
w.start_running()
if __name__ == '__main__':
skywriting.main("worker")
