def addHendrix(self):
"instantiates the HendrixService"
self.hendrix = HendrixService(
self.application, self.options['http_port'],
resources=self.resources, services=self.services,
loud=self.options['loud']
)
class HendrixDeploy(object):
"""
HendrixDeploy encapsulates the necessary information needed to deploy the
HendrixService on a single or multiple processes.
"""
def __init__(self, action='start', options={}, reactor=reactor):
self.action = action
self.options = hx_options()
self.options.update(options)
self.services = []
self.resources = []
self.reactor = reactor
self.use_settings = True # because running the management command overrides self.options['wsgi']
if self.options['wsgi']:
wsgi_dot_path = self.options['wsgi']
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.use_settings = False
else:
os.environ['DJANGO_SETTINGS_MODULE'] = self.options['settings']
django_conf = importlib.import_module('django.conf')
settings = getattr(django_conf, 'settings')
self.services = get_additional_services(settings)
self.resources = get_additional_resources(settings)
self.options = HendrixDeploy.getConf(settings, self.options)
if self.use_settings:
wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.is_secure = self.options['key'] and self.options['cert']
self.servers = []
@classmethod
def importWSGI(cls, wsgi_dot_path):
wsgi_module, application_name = wsgi_dot_path.rsplit('.', 1)
wsgi = importlib.import_module(wsgi_module)
return getattr(wsgi, application_name, None)
@classmethod
def getConf(cls, settings, options):
"updates the options dict to use config options in the settings module"
ports = ['http_port', 'https_port', 'cache_port']
for port_name in ports:
port = getattr(settings, port_name.upper(), None)
# only use the settings ports if the defaults were left unchanged
default = getattr(defaults, port_name.upper())
if port and options.get(port_name) == default:
options[port_name] = port
_opts = [('key', 'hx_private_key'), ('cert', 'hx_certficate'), ('wsgi', 'wsgi_application')]
for opt_name, settings_name in _opts:
opt = getattr(settings, settings_name.upper(), None)
if opt:
options[opt_name] = opt
if not options['settings']:
options['settings'] = environ['DJANGO_SETTINGS_MODULE']
return options
def addServices(self):
"""
a helper function used in HendrixDeploy.run
it instanstiates the HendrixService and adds child services
note that these services will also be run on all processes
"""
self.addHendrix()
if not self.options.get('global_cache') and not self.options.get('nocache'):
self.addLocalCacheService()
if self.is_secure:
self.addSSLService()
self.catalogServers(self.hendrix)
def addHendrix(self):
"instantiates the HendrixService"
self.hendrix = HendrixService(
self.application, self.options['http_port'], resources=self.resources,
services=self.services, loud=self.options['loud']
)
def catalogServers(self, hendrix):
"collects a list of service names serving on TCP or SSL"
for service in hendrix.services:
if isinstance(service, TCPServer) or isinstance(service, SSLServer):
self.servers.append(service.name)
def getCacheService(self):
cache_port = self.options.get('cache_port')
http_port = self.options.get('http_port')
return CacheService(
host='localhost', from_port=cache_port, to_port=http_port, path=''
)
def addLocalCacheService(self):
"adds a CacheService to the instatiated HendrixService"
_cache = self.getCacheService()
_cache.setName('cache_proxy')
_cache.setServiceParent(self.hendrix)
def addSSLService(self):
"adds a SSLService to the instaitated HendrixService"
https_port = self.options['https_port']
key = self.options['key']
cert = self.options['cert']
_tcp = self.hendrix.getServiceNamed('main_web_tcp')
factory = _tcp.factory
_ssl = ssl.SSLServer(https_port, factory, key, cert)
_ssl.setName('main_web_ssl')
_ssl.setServiceParent(self.hendrix)
def run(self):
"sets up the desired services and runs the requested action"
self.addServices()
action = self.action
fd = self.options['fd']
if action.startswith('start'):
chalk.blue('Ready and Listening...')
getattr(self, action)(fd)
self.reactor.run()
elif action == 'restart':
getattr(self, action)(fd=fd)
else:
getattr(self, action)()
@property
def pid(self):
"The default location of the pid file for process management"
return get_pid(self.options)
def getSpawnArgs(self):
"""
For the child processes we don't need to specify the SSL or caching
parameters as
"""
_args = [
executable, # path to python executable e.g. /usr/bin/python
]
if not self.options['loud']:
_args += ['-W', 'ignore',]
_args += [
'manage.py',
'hx',
'start',
'--http_port', str(self.options['http_port']),
'--https_port', str(self.options['https_port']),
'--cache_port', str(self.options['cache_port']),
'--workers', '0',
'--fd', pickle.dumps(self.fds),
]
if self.is_secure:
_args += [
'--key', self.options.get('key'),
'--cert', self.options.get('cert')
]
if self.options['nocache']:
_args.append('--nocache')
if self.options['dev']:
_args.append('--dev')
if self.options['traceback']:
_args.append('--traceback')
if self.options['global_cache']:
_args.append('--global_cache')
if not self.use_settings:
_args += ['--wsgi', self.options['wsgi']]
return _args
def addGlobalServices(self):
if self.options.get('global_cache') and not self.options.get('nocache'):
_cache = self.getCacheService()
_cache.startService()
def start(self, fd=None):
pids = [str(os.getpid())] # script pid
if fd is None:
# anything in this block is only run once
self.addGlobalServices()
self.hendrix.startService()
if self.options['workers']:
# Create a new listening port and several other processes to help out.
childFDs = {0: 0, 1: 1, 2: 2}
self.fds = {}
for name in self.servers:
port = self.hendrix.get_port(name)
fd = port.fileno()
childFDs[fd] = fd
self.fds[name] = fd
args = self.getSpawnArgs()
transports = []
for i in range(self.options['workers']):
transport = self.reactor.spawnProcess(
None, executable, args, childFDs=childFDs, env=environ
)
transports.append(transport)
pids.append(str(transport.pid))
with open(self.pid, 'w') as pid_file:
pid_file.write('\n'.join(pids))
else:
fds = pickle.loads(fd)
factories = {}
for name in self.servers:
factory = self.disownService(name)
factories[name] = factory
self.hendrix.startService()
for name, factory in factories.iteritems():
if name == 'main_web_ssl':
privateCert = PrivateCertificate.loadPEM(
open(self.options['cert']).read() + open(self.options['key']).read()
)
factory = TLSMemoryBIOFactory(
privateCert.options(), False, factory
)
port = self.reactor.adoptStreamPort(fds[name], AF_INET, factory)
def stop(self, sig=9):
with open(self.pid) as pid_file:
pids = pid_file.readlines()
for pid in pids:
try:
os.kill(int(pid), sig)
except OSError:
# OSError is raised when it trys to kill the child processes
pass
os.remove(self.pid)
def start_reload(self, fd=None):
self.start(fd=fd)
def restart(self, fd=None):
self.stop()
time.sleep(1) # wait a second to ensure the port is closed
self.start(fd)
def disownService(self, name):
"""
disowns a service on hendirix by name
returns a factory for use in the adoptStreamPort part of setting up
multiple processes
"""
_service = self.hendrix.getServiceNamed(name)
_service.disownServiceParent()
return _service.factory
class HendrixDeploy(object):
"""
HendrixDeploy encapsulates the necessary information needed to deploy
the HendrixService on a single or multiple processes.
"""
def __init__(self, action='start', options={}, reactor=reactor):
self.action = action
self.options = hx_options()
self.options.update(options)
self.services = []
self.resources = []
self.reactor = reactor
self.use_settings = True
# because running the management command overrides self.options['wsgi']
if self.options['wsgi']:
if hasattr(self.options['wsgi'], '__call__'):
# If it has a __call__, we assume that it is the application object itself.
self.application = self.options['wsgi']
try:
self.options['wsgi_app_name'] = "%s.%s" % (self.application.__module__, self.application.__name__)
except AttributeError:
self.options['wsgi_app_name'] = self.application.__class__.__name__
else:
# Otherwise, we'll try to discern an application in the belief that this is a dot path.
wsgi_dot_path = self.options['wsgi']
self.application = HendrixDeploy.importWSGI(wsgi_dot_path) # will raise AttributeError if we can't import it.
self.options['wsgi_app_name'] = wsgi_dot_path
self.use_settings = False
else:
os.environ['DJANGO_SETTINGS_MODULE'] = self.options['settings']
settings = import_string('django.conf.settings')
self.services = get_additional_services(settings)
self.resources = get_additional_resources(settings)
self.options = HendrixDeploy.getConf(settings, self.options)
if self.use_settings:
django = importlib.import_module('django')
if django.VERSION[:2] >= (1, 7):
django.setup()
wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.is_secure = self.options['key'] and self.options['cert']
self.servers = []
@classmethod
def importWSGI(cls, wsgi_dot_path):
try:
wsgi_module, application_name = wsgi_dot_path.rsplit('.', 1)
except AttributeError:
pid = os.getpid()
chalk.red(
"Unable to discern a WSGI application from '%s'" %
wsgi_dot_path
)
os.kill(pid, 15)
wsgi = importlib.import_module(wsgi_module)
return getattr(wsgi, application_name, None)
@classmethod
def getConf(cls, settings, options):
"updates the options dict to use config options in the settings module"
ports = ['http_port', 'https_port', 'cache_port']
for port_name in ports:
port = getattr(settings, port_name.upper(), None)
# only use the settings ports if the defaults were left unchanged
default = getattr(defaults, port_name.upper())
if port and options.get(port_name) == default:
options[port_name] = port
_opts = [
('key', 'hx_private_key'),
('cert', 'hx_certficate'),
('wsgi', 'wsgi_application')
]
for opt_name, settings_name in _opts:
opt = getattr(settings, settings_name.upper(), None)
if opt:
options[opt_name] = opt
if not options['settings']:
options['settings'] = environ['DJANGO_SETTINGS_MODULE']
return options
def addServices(self):
"""
a helper function used in HendrixDeploy.run
it instanstiates the HendrixService and adds child services
note that these services will also be run on all processes
"""
self.addHendrix()
def addHendrix(self):
"instantiates the HendrixService"
self.hendrix = HendrixService(
self.application, self.options['http_port'],
resources=self.resources, services=self.services,
loud=self.options['loud']
)
def catalogServers(self, hendrix):
"collects a list of service names serving on TCP or SSL"
for service in hendrix.services:
if isinstance(service, (TCPServer, SSLServer)):
self.servers.append(service.name)
def run(self):
"sets up the desired services and runs the requested action"
self.addServices()
self.catalogServers(self.hendrix)
action = self.action
fd = self.options['fd']
if action.startswith('start'):
chalk.blue(
'Ready and Listening on port %d...' % self.options.get(
'http_port'
)
)
getattr(self, action)(fd)
self.reactor.run()
elif action == 'restart':
getattr(self, action)(fd=fd)
else:
getattr(self, action)()
@property
def pid(self):
"The default location of the pid file for process management"
return get_pid(self.options)
def getSpawnArgs(self):
_args = [
'hx',
'start', # action
# kwargs
'--http_port', str(self.options['http_port']),
'--https_port', str(self.options['https_port']),
'--cache_port', str(self.options['cache_port']),
'--workers', '0',
'--fd', pickle.dumps(self.fds),
]
# args/signals
if self.options['dev']:
_args.append('--dev')
if self.options['traceback']:
_args.append('--traceback')
if not self.use_settings:
_args += ['--wsgi', self.options['wsgi']]
return _args
def addGlobalServices(self):
"""
This is where we put service that we don't want to be duplicated on
worker subprocesses
"""
pass
def start(self, fd=None):
if fd is None:
# anything in this block is only run once
self.addGlobalServices()
self.hendrix.startService()
self.launchWorkers()
else:
fds = pickle.loads(fd)
factories = {}
for name in self.servers:
factory = self.disownService(name)
factories[name] = factory
self.hendrix.startService()
for name, factory in factories.iteritems():
self.addSubprocesses(fds, name, factory)
def launchWorkers(self):
pids = [str(os.getpid())] # script pid
if self.options['workers']:
# Create a new listening port and several other processes to
# help out.
childFDs = {0: 0, 1: 1, 2: 2}
self.fds = {}
for name in self.servers:
port = self.hendrix.get_port(name)
fd = port.fileno()
childFDs[fd] = fd
self.fds[name] = fd
args = self.getSpawnArgs()
transports = []
for i in range(self.options['workers']):
transport = self.reactor.spawnProcess(
None, 'hx', args, childFDs=childFDs, env=environ
)
transports.append(transport)
pids.append(str(transport.pid))
with open(self.pid, 'w') as pid_file:
pid_file.write('\n'.join(pids))
def addSubprocesses(self, fds, name, factory):
self.reactor.adoptStreamPort( # outputs port
fds[name], AF_INET, factory
)
def stop(self, sig=9):
with open(self.pid) as pid_file:
pids = pid_file.readlines()
for pid in pids:
try:
os.kill(int(pid), sig)
except OSError:
# OSError raised when it trys to kill the child processes
pass
os.remove(self.pid)
def start_reload(self, fd=None):
self.start(fd=fd)
def restart(self, fd=None):
self.stop()
time.sleep(1) # wait a second to ensure the port is closed
self.start(fd)
def disownService(self, name):
"""
disowns a service on hendirix by name
returns a factory for use in the adoptStreamPort part of setting up
multiple processes
"""
_service = self.hendrix.getServiceNamed(name)
_service.disownServiceParent()
return _service.factory