def master_main(options):
deferred_worker = DeferredWorkPlugin(ciel.engine)
deferred_worker.subscribe()
worker_pool = WorkerPool(ciel.engine, deferred_worker, None)
worker_pool.subscribe()
task_failure_investigator = TaskFailureInvestigator(worker_pool, deferred_worker)
job_pool = JobPool(ciel.engine, options.journaldir, None, task_failure_investigator, deferred_worker, worker_pool)
job_pool.subscribe()
worker_pool.job_pool = job_pool
backup_sender = BackupSender(cherrypy.engine)
backup_sender.subscribe()
if options.hostname is not None:
local_hostname = options.hostname
else:
local_hostname = socket.getfqdn()
local_port = cherrypy.config.get('server.socket_port')
master_netloc = '%s:%d' % (local_hostname, local_port)
ciel.log('Local port is %d' % local_port, 'STARTUP', logging.INFO)
if options.blockstore is None:
static_content_root = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
else:
static_content_root = options.blockstore
block_store_dir = os.path.join(static_content_root, "data")
try:
os.mkdir(block_store_dir)
except:
pass
block_store = BlockStore(ciel.engine, local_hostname, local_port, block_store_dir)
block_store.subscribe()
block_store.build_pin_set()
block_store.check_local_blocks()
if options.master is not None:
monitor = MasterRecoveryMonitor(cherrypy.engine, 'http://%s/' % master_netloc, options.master, job_pool)
monitor.subscribe()
else:
monitor = None
recovery_manager = RecoveryManager(ciel.engine, job_pool, block_store, deferred_worker)
recovery_manager.subscribe()
root = MasterRoot(worker_pool, block_store, job_pool, backup_sender, monitor)
cherrypy.config.update({"server.thread_pool" : 50})
cherrypy_conf = dict()
if options.staticbase is not None:
cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase }
app = cherrypy.tree.mount(root, "", cherrypy_conf)
lighty_conf_template = options.lighty_conf
if lighty_conf_template is not None:
lighty = LighttpdAdapter(ciel.engine, lighty_conf_template, static_content_root, local_port)
lighty.subscribe()
# Zap CherryPy's original flavour server
cherrypy.server.unsubscribe()
server = cherrypy.process.servers.FlupFCGIServer(application=app, bindAddress=lighty.socket_path)
adapter = cherrypy.process.servers.ServerAdapter(cherrypy.engine, httpserver=server, bind_addr=lighty.socket_path)
# Insert a FastCGI server in its place
adapter.subscribe()
if hasattr(ciel.engine, "signal_handler"):
ciel.engine.signal_handler.subscribe()
if hasattr(ciel.engine, "console_control_handler"):
ciel.engine.console_control_handler.subscribe()
ciel.engine.start()
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
post_string(urllib2.urlparse.urljoin(worker_url, 'control/master/'), master_details_as_json)
# Worker will be created by a callback.
except:
ciel.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING)
ciel.engine.block()
def master_main(options):
deferred_worker = DeferredWorkPlugin(cherrypy.engine)
deferred_worker.subscribe()
global_name_directory = GlobalNameDirectory(cherrypy.engine)
global_name_directory.subscribe()
worker_pool = WorkerPool(cherrypy.engine, deferred_worker)
worker_pool.subscribe()
lazy_task_pool = LazyTaskPool(cherrypy.engine, worker_pool)
task_pool_adapter = LazyTaskPoolAdapter(lazy_task_pool)
lazy_task_pool.subscribe()
task_failure_investigator = TaskFailureInvestigator(lazy_task_pool, worker_pool, deferred_worker)
job_pool = JobPool(cherrypy.engine, lazy_task_pool, options.journaldir, global_name_directory)
job_pool.subscribe()
backup_sender = BackupSender(cherrypy.engine)
backup_sender.subscribe()
local_hostname = socket.getfqdn()
local_port = cherrypy.config.get('server.socket_port')
master_netloc = '%s:%d' % (local_hostname, local_port)
print 'Local port is', local_port
if options.blockstore is None:
block_store_dir = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
else:
block_store_dir = options.blockstore
block_store = BlockStore(cherrypy.engine, local_hostname, local_port, block_store_dir)
block_store.build_pin_set()
if options.master is not None:
monitor = MasterRecoveryMonitor(cherrypy.engine, 'http://%s/' % master_netloc, options.master, job_pool)
monitor.subscribe()
else:
monitor = None
recovery_manager = RecoveryManager(cherrypy.engine, job_pool, lazy_task_pool, block_store, deferred_worker)
recovery_manager.subscribe()
scheduler = LazyScheduler(cherrypy.engine, lazy_task_pool, worker_pool)
scheduler.subscribe()
root = MasterRoot(task_pool_adapter, worker_pool, block_store, global_name_directory, job_pool, backup_sender, monitor, task_failure_investigator)
cherrypy.config.update({"server.thread_pool" : 50})
cherrypy_conf = dict()
if options.staticbase is not None:
cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase }
cherrypy.tree.mount(root, "", cherrypy_conf)
if hasattr(cherrypy.engine, "signal_handler"):
cherrypy.engine.signal_handler.subscribe()
if hasattr(cherrypy.engine, "console_control_handler"):
cherrypy.engine.console_control_handler.subscribe()
cherrypy.engine.start()
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
http = httplib2.Http()
http.request(urllib2.urlparse.urljoin(worker_url, '/master/'), "POST", master_details_as_json)
# Worker will be created by a callback.
except:
cherrypy.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING)
cherrypy.engine.block()
def master_main(options):
create_pycurl_thread(ciel.engine)
deferred_worker = DeferredWorkPlugin(ciel.engine)
deferred_worker.subscribe()
worker_pool = WorkerPool(ciel.engine, deferred_worker, None)
worker_pool.subscribe()
task_failure_investigator = TaskFailureInvestigator(worker_pool, deferred_worker)
job_pool = JobPool(ciel.engine, options.journaldir, None, task_failure_investigator, deferred_worker, worker_pool)
job_pool.subscribe()
worker_pool.job_pool = job_pool
backup_sender = BackupSender(cherrypy.engine)
backup_sender.subscribe()
if options.hostname is not None:
local_hostname = options.hostname
else:
local_hostname = socket.getfqdn()
local_port = cherrypy.config.get('server.socket_port')
master_netloc = '%s:%d' % (local_hostname, local_port)
ciel.log('Local port is %d' % local_port, 'STARTUP', logging.INFO)
if options.blockstore is None:
static_content_root = tempfile.mkdtemp(prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
else:
static_content_root = options.blockstore
block_store_dir = os.path.join(static_content_root, "data")
try:
os.mkdir(block_store_dir)
except:
pass
block_store = BlockStore(local_hostname, local_port, block_store_dir)
block_store.build_pin_set()
block_store.check_local_blocks()
if options.master is not None:
monitor = MasterRecoveryMonitor(cherrypy.engine, 'http://%s/' % master_netloc, options.master, job_pool)
monitor.subscribe()
else:
monitor = None
recovery_manager = RecoveryManager(ciel.engine, job_pool, block_store, deferred_worker)
recovery_manager.subscribe()
root = MasterRoot(worker_pool, block_store, job_pool, backup_sender, monitor)
cherrypy.config.update({"server.thread_pool" : 50})
cherrypy_conf = dict()
if options.staticbase is not None:
cherrypy_conf["/skyweb"] = { "tools.staticdir.on": True, "tools.staticdir.dir": options.staticbase }
app = cherrypy.tree.mount(root, "", cherrypy_conf)
lighty_conf_template = options.lighty_conf
if lighty_conf_template is not None:
lighty = LighttpdAdapter(ciel.engine, lighty_conf_template, static_content_root, local_port)
lighty.subscribe()
# Zap CherryPy's original flavour server
cherrypy.server.unsubscribe()
server = cherrypy.process.servers.FlupFCGIServer(application=app, bindAddress=lighty.socket_path)
adapter = cherrypy.process.servers.ServerAdapter(cherrypy.engine, httpserver=server, bind_addr=lighty.socket_path)
# Insert a FastCGI server in its place
adapter.subscribe()
if hasattr(ciel.engine, "signal_handler"):
ciel.engine.signal_handler.subscribe()
if hasattr(ciel.engine, "console_control_handler"):
ciel.engine.console_control_handler.subscribe()
ciel.engine.start()
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
post_string(urllib2.urlparse.urljoin(worker_url, 'control/master/'), master_details_as_json)
# Worker will be created by a callback.
except:
ciel.log.error("Error adding worker: %s" % (worker_url, ), "WORKER", logging.WARNING)
ciel.engine.block()
def master_main(options):
deferred_worker = DeferredWorkPlugin(cherrypy.engine)
deferred_worker.subscribe()
global_name_directory = GlobalNameDirectory(cherrypy.engine)
global_name_directory.subscribe()
worker_pool = WorkerPool(cherrypy.engine, deferred_worker)
worker_pool.subscribe()
lazy_task_pool = LazyTaskPool(cherrypy.engine, worker_pool)
task_pool_adapter = LazyTaskPoolAdapter(lazy_task_pool)
lazy_task_pool.subscribe()
task_failure_investigator = TaskFailureInvestigator(
lazy_task_pool, worker_pool, deferred_worker)
job_pool = JobPool(cherrypy.engine, lazy_task_pool, options.journaldir,
global_name_directory)
job_pool.subscribe()
backup_sender = BackupSender(cherrypy.engine)
backup_sender.subscribe()
local_hostname = socket.getfqdn()
local_port = cherrypy.config.get('server.socket_port')
master_netloc = '%s:%d' % (local_hostname, local_port)
print 'Local port is', local_port
if options.blockstore is None:
block_store_dir = tempfile.mkdtemp(
prefix=os.getenv('TEMP', default='/tmp/sw-files-'))
else:
block_store_dir = options.blockstore
block_store = BlockStore(cherrypy.engine, local_hostname, local_port,
block_store_dir)
block_store.build_pin_set()
if options.master is not None:
monitor = MasterRecoveryMonitor(cherrypy.engine,
'http://%s/' % master_netloc,
options.master, job_pool)
monitor.subscribe()
else:
monitor = None
recovery_manager = RecoveryManager(cherrypy.engine, job_pool,
lazy_task_pool, block_store,
deferred_worker)
recovery_manager.subscribe()
scheduler = LazyScheduler(cherrypy.engine, lazy_task_pool, worker_pool)
scheduler.subscribe()
root = MasterRoot(task_pool_adapter, worker_pool, block_store,
global_name_directory, job_pool, backup_sender, monitor,
task_failure_investigator)
cherrypy.config.update({"server.thread_pool": 50})
cherrypy_conf = dict()
if options.staticbase is not None:
cherrypy_conf["/skyweb"] = {
"tools.staticdir.on": True,
"tools.staticdir.dir": options.staticbase
}
cherrypy.tree.mount(root, "", cherrypy_conf)
if hasattr(cherrypy.engine, "signal_handler"):
cherrypy.engine.signal_handler.subscribe()
if hasattr(cherrypy.engine, "console_control_handler"):
cherrypy.engine.console_control_handler.subscribe()
cherrypy.engine.start()
if options.workerlist is not None:
master_details = {'netloc': master_netloc}
master_details_as_json = simplejson.dumps(master_details)
with (open(options.workerlist, "r")) as f:
for worker_url in f.readlines():
try:
http = httplib2.Http()
http.request(
urllib2.urlparse.urljoin(worker_url, '/master/'),
"POST", master_details_as_json)
# Worker will be created by a callback.
except:
cherrypy.log.error(
"Error adding worker: %s" % (worker_url, ), "WORKER",
logging.WARNING)
cherrypy.engine.block()
