def __init__(self, config):
self.config = config
self.host = config.get('controller', 'host', default='localhost')
self.port = config.getint('controller', 'port', default=8888)
self.proxy = config.getboolean('controller', 'proxy', default=False)
if self.proxy:
self.proxy_connect_port = config.getint('controller',
'proxy_connect_port')
self.ssl = config.getboolean('controller', 'ssl', default=True)
self.data_dir = config.get('controller', 'data-dir',
default=self.DEFAULT_DATA_DIR)
if not os.path.isdir(self.data_dir):
os.mkdir(self.data_dir)
for sub_dir in ["data", "archive", "logs/archive"]:
tdir = os.path.join(self.data_dir, sub_dir)
if not os.path.isdir(tdir):
os.makedirs(tdir)
self.uuid = config.get(self.DEFAULT_SECTION, 'uuid')
self.version = version()
self.license_key = config.get(self.DEFAULT_SECTION, 'license-key',
default=self.DEFAULT_LICENSE_KEY)
self.archive_port = config.get('archive', 'port', default=8889)
self.install_dir = os.path.abspath(os.path.dirname(__file__))
self.xid_dir = config.get(self.DEFAULT_SECTION, 'xid-dir',
default=None)
if not self.xid_dir:
self.xid_dir = os.path.join(self.install_dir, 'xid')
if not os.path.isdir(self.xid_dir):
os.mkdir(self.xid_dir)
pathenv = config.get(self.DEFAULT_SECTION, 'path', default=None)
self.processmanager = ProcessManager(self.xid_dir, pathenv)
conf = os.path.join(self.install_dir, 'conf', 'archive', 'httpd.conf')
port = config.getint("archive", "port", default=8889);
self.archive = Apache2(conf, port, self.data_dir)
def run(exedir, libdir, configdir, config_file, verbose):
config = ConfigParser.ConfigParser()
config.read([config_file])
service_names = config.get("runner", "services").split(",")
http_port = parse_port(config.get("runner", "http_port"))
https_ports = list()
if config.has_option("runner", "https_ports"):
for p in config.get("runner", "https_ports").split(","):
if not p:
continue
https_ports.append(parse_port(p))
if config.has_option("global", "rundir"):
rundir = config.get("global", "rundir")
else:
print 'warning: rundir in [runner] section is deprecated. put in [global]'
rundir = config.get("runner", "rundir")
if not os.path.isabs(rundir):
rundir = os.path.abspath(rundir)
logdir = config.get("runner", "logdir")
if not os.path.isabs(logdir):
logdir = os.path.abspath(logdir)
m2abin = "m2adapter"
path = os.path.normpath(os.path.join(exedir, "bin/m2adapter"))
if os.path.isfile(path):
m2abin = path
proxybin = "pushpin-proxy"
path = os.path.normpath(os.path.join(exedir, "bin/pushpin-proxy"))
if os.path.isfile(path):
proxybin = path
handlerbin = "pushpin-handler"
path = os.path.normpath(os.path.join(exedir, "bin/pushpin-handler"))
if os.path.isfile(path):
handlerbin = path
# make run/log dirs if needed. don't fail if dirs already exist
trymakedir(rundir)
trymakedir(logdir)
service_objs = list()
if "mongrel2" in service_names:
mongrel2_bin = "mongrel2"
if config.has_option("runner", "mongrel2_bin"):
mongrel2_bin = config.get("runner", "mongrel2_bin")
m2sh_bin = "m2sh"
if config.has_option("runner", "m2sh_bin"):
m2sh_bin = config.get("runner", "m2sh_bin")
m2sqlpath = services.write_mongrel2_config(configdir, os.path.join(libdir, "mongrel2.conf.template"), rundir, logdir, http_port, https_ports, m2sh_bin)
service_objs.append(services.Mongrel2Service(mongrel2_bin, m2sqlpath, False, http_port[1], rundir, logdir))
for port in https_ports:
service_objs.append(services.Mongrel2Service(mongrel2_bin, m2sqlpath, True, port[1], rundir, logdir))
if "m2adapter" in service_names:
ports = list()
ports.append(http_port[1])
ports.extend([port[1] for port in https_ports])
services.write_m2adapter_config(os.path.join(libdir, "m2adapter.conf.template"), rundir, ports)
service_objs.append(services.M2AdapterService(m2abin, os.path.join(rundir, "m2adapter.conf"), verbose, rundir, logdir))
if "zurl" in service_names:
services.write_zurl_config(os.path.join(libdir, "zurl.conf.template"), rundir)
zurl_bin = "zurl"
if config.has_option("runner", "zurl_bin"):
zurl_bin = config.get("runner", "zurl_bin")
service_objs.append(services.ZurlService(zurl_bin, os.path.join(rundir, "zurl.conf"), verbose, rundir, logdir))
if "pushpin-proxy" in service_names:
service_objs.append(services.PushpinProxyService(proxybin, config_file, verbose, rundir, logdir))
if "pushpin-handler" in service_names:
service_objs.append(services.PushpinHandlerService(handlerbin, config_file, verbose, rundir, logdir))
print "starting..."
p = ProcessManager()
p.reloadmessage = "reloading"
p.stopmessage = "stopping..."
for s in service_objs:
s.pre_start()
pid = p.add(s.name(), s.getargs(), s.getlogfile(), s.accept_sighup())
s.post_start(pid)
print "started"
p.wait()
for s in service_objs:
s.post_stop()
print "stopped"
import sys
if sys.version_info < (3, 3):
sys.stdout.write("Python 3.3 required\n")
sys.exit(1)
from multiprocessing import Process, Pipe
from multiprocessing.connection import wait
from processmanager import ProcessManager
def do_things_in_process(things_to_do = []):
return [i ** 12 for i in things_to_do]
process_manager = ProcessManager(2, do_things_in_process)
print(process_manager.get_their_work([0,1,2,3]))
print(process_manager.get_their_work([0,1,2,3]))
process_manager.stop()
process_manager = ProcessManager(2, do_things_in_process)
print(process_manager.get_their_work([4,5,6,7]))
print(process_manager.get_their_work([4,5,6,7]))
process_manager.stop()
def run(exedir, config_file, verbose):
config = ConfigParser.ConfigParser()
config.read([config_file])
configdir = config.get("runner", "configdir")
if not os.path.isabs(configdir):
configdir = os.path.join(os.path.dirname(config_file), configdir)
service_names = config.get("runner", "services").split(",")
http_port = int(config.get("runner", "http_port"))
https_ports = list()
if config.has_option("runner", "https_ports"):
for p in config.get("runner", "https_ports").split(","):
if not p:
continue
https_ports.append(int(p))
rundir = config.get("runner", "rundir")
if not os.path.isabs(rundir):
rundir = os.path.join(os.path.dirname(config_file), rundir)
logdir = config.get("runner", "logdir")
if not os.path.isabs(logdir):
logdir = os.path.join(os.path.dirname(config_file), logdir)
proxybin = "pushpin-proxy"
path = os.path.normpath(os.path.join(exedir, "proxy/pushpin-proxy"))
if os.path.isfile(path):
proxybin = path
handlerbin = "pushpin-handler"
path = os.path.normpath(os.path.join(exedir, "handler/pushpin-handler"))
if os.path.isfile(path):
handlerbin = path
service_objs = list()
if "mongrel2" in service_names:
mongrel2_bin = "mongrel2"
if config.has_option("runner", "mongrel2_bin"):
mongrel2_bin = config.get("runner", "mongrel2_bin")
m2sh_bin = "m2sh"
if config.has_option("runner", "m2sh_bin"):
m2sh_bin = config.get("runner", "m2sh_bin")
m2sqlpath = services.write_mongrel2_config(configdir, os.path.join(configdir, "mongrel2.conf.template"), rundir, http_port, https_ports, m2sh_bin)
service_objs.append(services.Mongrel2Service(mongrel2_bin, m2sqlpath, False, http_port, rundir, logdir))
for port in https_ports:
service_objs.append(services.Mongrel2Service(mongrel2_bin, m2sqlpath, True, port, rundir, logdir))
if "zurl" in service_names:
zurl_bin = "zurl"
if config.has_option("runner", "zurl_bin"):
zurl_bin = config.get("runner", "zurl_bin")
service_objs.append(services.ZurlService(zurl_bin, os.path.join(configdir, "zurl.conf"), verbose, rundir, logdir))
if "pushpin-proxy" in service_names:
service_objs.append(services.PushpinProxyService(proxybin, config_file, verbose, rundir, logdir))
if "pushpin-handler" in service_names:
service_objs.append(services.PushpinHandlerService(handlerbin, config_file, rundir, logdir))
print "starting..."
p = ProcessManager()
p.stopmessage = "stopping..."
for s in service_objs:
s.pre_start()
pid = p.add(s.name(), s.getargs(), s.getlogfile())
s.post_start(pid)
print "started"
p.wait()
for s in service_objs:
s.post_stop()
print "stopped"
def run(exedir, config_file, verbose):
config = ConfigParser.ConfigParser()
config.read([config_file])
configdir = config.get("runner", "configdir")
if not os.path.isabs(configdir):
configdir = os.path.abspath(configdir)
service_names = config.get("runner", "services").split(",")
http_port = int(config.get("runner", "http_port"))
https_ports = list()
if config.has_option("runner", "https_ports"):
for p in config.get("runner", "https_ports").split(","):
if not p:
continue
https_ports.append(int(p))
if config.has_option("global", "rundir"):
rundir = config.get("global", "rundir")
else:
print 'warning: rundir in [runner] section is deprecated. put in [global]'
rundir = config.get("runner", "rundir")
if not os.path.isabs(rundir):
rundir = os.path.abspath(rundir)
logdir = config.get("runner", "logdir")
if not os.path.isabs(logdir):
logdir = os.path.abspath(logdir)
m2abin = "m2adapter"
path = os.path.normpath(os.path.join(exedir, "m2adapter/m2adapter"))
if os.path.isfile(path):
m2abin = path
proxybin = "pushpin-proxy"
path = os.path.normpath(os.path.join(exedir, "proxy/pushpin-proxy"))
if os.path.isfile(path):
proxybin = path
handlerbin = "pushpin-handler"
path = os.path.normpath(os.path.join(exedir, "handler/pushpin-handler"))
if os.path.isfile(path):
handlerbin = path
# make run/log dirs if needed. don't fail if dirs already exist
trymakedir(rundir)
trymakedir(logdir)
service_objs = list()
if "mongrel2" in service_names:
mongrel2_bin = "mongrel2"
if config.has_option("runner", "mongrel2_bin"):
mongrel2_bin = config.get("runner", "mongrel2_bin")
m2sh_bin = "m2sh"
if config.has_option("runner", "m2sh_bin"):
m2sh_bin = config.get("runner", "m2sh_bin")
m2sqlpath = services.write_mongrel2_config(
configdir, os.path.join(configdir, "mongrel2.conf.template"),
rundir, logdir, http_port, https_ports, m2sh_bin)
service_objs.append(
services.Mongrel2Service(mongrel2_bin, m2sqlpath, False, http_port,
rundir, logdir))
for port in https_ports:
service_objs.append(
services.Mongrel2Service(mongrel2_bin, m2sqlpath, True, port,
rundir, logdir))
if "m2adapter" in service_names:
ports = list()
ports.append(http_port)
ports.extend(https_ports)
services.write_m2adapter_config(
os.path.join(configdir, "m2adapter.conf.template"), rundir, ports)
service_objs.append(
services.M2AdapterService(m2abin,
os.path.join(rundir, "m2adapter.conf"),
verbose, rundir, logdir))
if "zurl" in service_names:
services.write_zurl_config(
os.path.join(configdir, "zurl.conf.template"), rundir)
zurl_bin = "zurl"
if config.has_option("runner", "zurl_bin"):
zurl_bin = config.get("runner", "zurl_bin")
service_objs.append(
services.ZurlService(zurl_bin, os.path.join(rundir, "zurl.conf"),
verbose, rundir, logdir))
if "pushpin-proxy" in service_names:
service_objs.append(
services.PushpinProxyService(proxybin, config_file, verbose,
rundir, logdir))
if "pushpin-handler" in service_names:
service_objs.append(
services.PushpinHandlerService(handlerbin, config_file, verbose,
rundir, logdir))
print "starting..."
p = ProcessManager()
p.reloadmessage = "reloading"
p.stopmessage = "stopping..."
for s in service_objs:
s.pre_start()
pid = p.add(s.name(), s.getargs(), s.getlogfile(), s.accept_sighup())
s.post_start(pid)
print "started"
p.wait()
for s in service_objs:
s.post_stop()
print "stopped"
def compute_recreate(bugs, nb_process=1, repeat=1):
process_manager = ProcessManager(nb_process, run_worker)
for i in range(repeat):
process_manager.get_their_work(bugs)
