def update_config_file():
template = 'templates/supervisor/supervisor.conf'
group_template = 'templates/supervisor/supervisor-group.conf'
output = ''
groups = {}
for site, site_config in config.sites.items():
group_name = get_group_name(site, site_config)
groups[group_name] = []
for process_name, process_dict in site_config.processes.items():
context_dict = copy(site_config)
env_dict = {
'HOME': site_config.deployment['home'],
'PYTHONPATH': site_config.deployment['path'],
}
env_dict.update(site_config.environment)
context_dict.update({
'process_name': process_name,
'process_cmd': process_dict["command"],
'socket': process_dict["socket"],
'env': env_dict,
'priority': process_dict.get('priority', 200),
})
output += render_template(template, context_dict)
groups[group_name].append(process_name)
output += render_template(group_template, {'groups': groups})
put(StringIO.StringIO(output), '%(path)s/../config/supervisor.conf' % config.sites["main"].deployment)
update()
def update_config_file(dryrun=False, update_command=update, load_config=True):
output = ''
groups = {}
for site, site_config in config.sites.items():
template_path = site_config['supervisor']['template']
group_template_path = site_config['supervisor']['group_template']
group_name = get_group_name(site, site_config)
groups[group_name] = []
for process_name, process_dict in site_config.processes.items():
context_dict = copy(site_config)
env_dict = {
'HOME': site_config.deployment['home'],
'USER': site_config.deployment['user'],
'PYTHONPATH': ":".join([
site_config.deployment['path'],
posixpath.join(site_config.deployment['path'], site_config.get("django").get("base")+'/'),
]),
}
env_dict.update(site_config.environment)
context_dict.update({
'process_name': process_name,
'process_cmd': process_dict["command"],
'socket': process_dict["socket"],
'env': env_dict,
'priority': process_dict.get('priority', 200),
'autostart': 'True' if getattr(env, 'autostart', True) else 'False',
'killasgroup': process_dict.get('killasgroup', None),
'stopasgroup': process_dict.get('killasgroup', None),
'stopwaitsecs': process_dict.get('stopwaitsecs', None),
})
output += render_template(template_path, context_dict)
groups[group_name].append(process_name)
output += render_template(group_template_path, {'groups': groups})
conf_path = posixpath.abspath(posixpath.join(config.sites["main"].deployment['path'], '..', 'config'))
path = posixpath.join(conf_path, 'supervisor.conf')
daemon_template_path = site_config['supervisor']['daemon_template']
supervisord_conf_path = posixpath.join(conf_path, 'supervisord.conf')
supervisord_conf_output = render_template(daemon_template_path, copy(site_config))
if dryrun:
print path + ':'
print output
print daemon_template_path + ':'
print supervisord_conf_output
else:
safe_put(StringIO.StringIO(output), path)
safe_put(StringIO.StringIO(supervisord_conf_output), supervisord_conf_path)
if load_config:
update_command()
def update_config_file(dryrun=False):
for site, site_config in list(config.sites.items()):
redis_processes = [(x, site_config.processes[x])
for x in site_config.processes
if site_config.processes[x]["type"] == "redis"]
template_path = site_config['redis']['template']
print(redis_processes)
for process_name, process in redis_processes:
working_directoy = posixpath.normpath(
posixpath.join(env.path, '..', 'data', 'redis', process_name))
log_directory = posixpath.normpath(
posixpath.join(site_config['deployment']['logdir'], 'log',
'redis'))
run('mkdir -p ' + working_directoy)
run('mkdir -p ' + log_directory)
context_dict = site_config
context_dict.update({
'site': site,
'working_directory': working_directoy,
'log_directory': log_directory,
'process_name': process_name,
'socket': process['socket'],
})
path = posixpath.abspath(
posixpath.join(site_config['deployment']['path'], '..',
'config', process_name + '.conf'))
output = render_template(template_path, context_dict)
if dryrun:
print((path + ":"))
print(output)
else:
put(io.StringIO(output), path)
def update_config_file(dryrun=False):
for site, site_config in config.sites.items():
redis_processes = [
(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"
]
template_path = site_config["redis"]["template"]
print redis_processes
for process_name, process in redis_processes:
working_directoy = posixpath.normpath(posixpath.join(env.path, "..", "data", "redis", process_name))
log_directory = posixpath.normpath(posixpath.join(env.path, "..", "log", "redis"))
run("mkdir -p " + working_directoy)
run("mkdir -p " + log_directory)
context_dict = site_config
context_dict.update(
{
"site": site,
"working_directory": working_directoy,
"log_directory": log_directory,
"process_name": process_name,
"socket": process["socket"],
}
)
path = posixpath.abspath(
posixpath.join(site_config["deployment"]["path"], "..", "config", process_name + ".conf")
)
output = render_template(template_path, context_dict)
if dryrun:
print path + ":"
print output
else:
put(StringIO.StringIO(output), path)
def update_config_file(dryrun=False):
for site, site_config in config.sites.items():
redis_processes = [(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"]
template_path = site_config['redis']['template']
print redis_processes
for process_name, process in redis_processes:
working_directoy = posixpath.normpath(posixpath.join(env.path, '..', 'data', 'redis', process_name))
log_directory = posixpath.normpath(posixpath.join(site_config['deployment']['logdir'], 'log', 'redis'))
run('mkdir -p ' + working_directoy)
run('mkdir -p ' + log_directory)
context_dict = site_config
context_dict.update({
'site': site,
'working_directory': working_directoy,
'log_directory': log_directory,
'process_name': process_name,
'socket': process['socket'],
})
path = posixpath.abspath(posixpath.join(site_config['deployment']['path'], '..', 'config', process_name + '.conf'))
output = render_template(template_path, context_dict)
if dryrun:
print path + ":"
print output
else:
put(StringIO.StringIO(output), path)
def update_config_file(dryrun=False, update_command=update):
output = ""
groups = {}
for site, site_config in config.sites.items():
template_path = site_config["supervisor"]["template"]
group_template_path = site_config["supervisor"]["group_template"]
group_name = get_group_name(site, site_config)
groups[group_name] = []
for process_name, process_dict in site_config.processes.items():
context_dict = copy(site_config)
env_dict = {
"HOME": site_config.deployment["home"],
"USER": site_config.deployment["user"],
"PYTHONPATH": ":".join(
[
site_config.deployment["path"],
posixpath.join(site_config.deployment["path"], site_config.get("django").get("base") + "/"),
]
),
}
env_dict.update(site_config.environment)
context_dict.update(
{
"process_name": process_name,
"process_cmd": process_dict["command"],
"socket": process_dict["socket"],
"env": env_dict,
"priority": process_dict.get("priority", 200),
"autostart": "True" if getattr(env, "autostart", True) else "False",
"killasgroup": process_dict.get("killasgroup", None),
"stopasgroup": process_dict.get("killasgroup", None),
"stopwaitsecs": process_dict.get("stopwaitsecs", None),
}
)
output += render_template(template_path, context_dict)
groups[group_name].append(process_name)
output += render_template(group_template_path, {"groups": groups})
path = posixpath.abspath(posixpath.join(config.sites["main"].deployment["path"], "..", "config", "supervisor.conf"))
if dryrun:
print path + ":"
print output
else:
put(StringIO.StringIO(output), path)
update_command()
def update_config_file():
template = 'templates/supervisor/supervisor.conf'
output = ''
for site, site_config in config.sites.items():
for process_name, process_dict in site_config.processes.items():
context_dict = site_config
context_dict.update({'process_name': process_name, 'process_cmd': process_dict["command"], 'socket': process_dict["socket"]})
output += render_template(template, context_dict)
put(StringIO.StringIO(output), '%(path)s/../config/supervisor.conf' % config.sites["main"].deployment)
update()
def update_config_file():
output = ""
template_name = 'templates/nginx/nginx.conf'
for site, site_config in config.sites.items():
context_dict = site_config
context_dict.update({
'domains': " ".join(site_config.deployment.get("domains")[site]),
'www_processes': [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"],
})
output += render_template(template_name, context_dict)
put(StringIO.StringIO(output), '%(path)s/../config/nginx.conf' % env)
reload_nginx()
def update_config_file():
template = 'templates/supervisor/supervisor.conf'
output = ''
for site, site_config in config.sites.items():
for process_name, process_dict in site_config.processes.items():
context_dict = site_config
context_dict.update({
'process_name': process_name,
'process_cmd': process_dict["command"],
'socket': process_dict["socket"]
})
output += render_template(template, context_dict)
put(StringIO.StringIO(output),
'%(path)s/../config/supervisor.conf' % config.sites["main"].deployment)
update()
def update_config_file(dryrun=False):
output = ""
for site, site_config in config.sites.items():
context_dict = site_config
context_dict.update({
'domains': " ".join(site_config.deployment.get("domains")[site]),
'www_processes': [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"],
})
template_path = context_dict['nginx']['template']
output += render_template(template_path, context_dict)
path = posixpath.abspath(posixpath.join(env.path, '..', 'config', 'nginx.conf'))
if dryrun:
print path + ':'
print output
else:
put(StringIO.StringIO(output), path)
reload_nginx()
def update_config_file():
template_name = 'templates/redis/redis.conf'
for site, site_config in config.sites.items():
redis_processes = [(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"]
print redis_processes
for process_name, process in redis_processes:
working_directoy = posixpath.normpath(posixpath.join(env.path, '..', 'data', 'redis', process_name))
log_directory = posixpath.normpath(posixpath.join(env.path, '..', 'log', 'redis'))
run('mkdir -p ' + working_directoy)
run('mkdir -p ' + log_directory)
context_dict = site_config
context_dict.update({
'site': site,
'working_directory': working_directoy,
'log_directory': log_directory,
'process_name': process_name,
'socket': process['socket'],
})
output = render_template(template_name, context_dict)
put(StringIO.StringIO(output), posixpath.join(site_config['deployment']['path'], '..', 'config', process_name + '.conf'))
def update_config_file(dryrun=False):
output = ""
for site, site_config in config.sites.items():
context_dict = site_config
context_dict.update({
'domains':
" ".join(site_config.deployment.get("domains")[site]),
'www_processes': [
site_config.processes[x] for x in site_config.processes
if site_config.processes[x]["type"] == "gunicorn"
],
})
template_path = context_dict['nginx']['template']
output += render_template(template_path, context_dict)
path = posixpath.abspath(
posixpath.join(env.path, '..', 'config', 'nginx.conf'))
if dryrun:
print path + ':'
print output
else:
put(StringIO.StringIO(output), path)
reload_nginx()
def processes(self, site, env_dict):
"""
Returns a dictionary of dictionaries each having the following keys:
* command
command to be run by supervisor
* port
port number,
* socket
path to unix socket
* type
gunicorn/memcached/celeryd
"""
process_dict = {}
site_dict = self.sites[site]
common_cmd_context = {
"django_cmd": site_dict.django['cmd'],
"django_args": " ".join(site_dict.get("django").get("args", [])),
}
socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_gunicorn.sock" % (env_dict.get("user"), site))) # Asserts pony project layout
if site_dict.gunicorn['bind']:
bind = site_dict.gunicorn['bind']
else:
bind = 'unix:{}'.format(socket)
cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path"))
cmd += 'bin/gunicorn'
gunicorn_cmd_context = {
'cmd': cmd,
"socket": socket,
"bind": bind,
"workers": site_dict.gunicorn['workers'],
"maxrequests": site_dict.gunicorn['maxrequests'],
"timeout": site_dict.gunicorn['timeout'],
"version": site_dict.gunicorn['version'],
}
gunicorn_cmd_context.update(common_cmd_context)
gunicorn_command_template_path = self.sites[site]['supervisor']['gunicorn_command_template']
gunicorn_command = render_template(
gunicorn_command_template_path,
gunicorn_cmd_context,
strip_newlines=True,
)
process_dict["%s_%s_gunicorn" % (env_dict.get("user"), site)] = {
'command': gunicorn_command,
'port': None,
'socket': gunicorn_cmd_context['socket'],
'type': 'gunicorn',
'priority': 100,
}
custom_processes = env_dict.get("custom_processes") or []
for process in custom_processes:
process_config = site_dict[process]
if not process_config.get("enabled"):
continue
custom_command = process_config['command']
if process_config.get('django'):
custom_command = '%s %s' % (site_dict.django['cmd'], custom_command)
process_name = "%s_%s_%s" % (env_dict.get("user"), site, process)
process_dict[process_name] = {
'command': custom_command,
'type': 'custom',
'priority': 100,
'port': None,
'socket': None,
}
if site_dict.get("memcached").get("enabled"):
memcached_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_memcached.sock" % (env_dict.get("user"), site))) # Asserts pony project layout
process_dict["%s_%s_memcached" % (env_dict.get("user"), site)] = {
'command': "memcached -s %s -m %d" % (memcached_socket, int(site_dict.get("memcached").get("size"))),
'port': None,
'socket': memcached_socket,
'type': 'memcached',
'priority': 60,
}
if site_dict.get("celery").get("enabled"):
conf = site_dict.get("celery")
cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path"))
cmd += 'bin/celery'
celeryd_command_context = {
'concurrency': conf.get("concurrency"),
'maxtasksperchild': conf.get("maxtasksperchild"),
'loglevel': conf.get("loglevel"),
'extra_options': conf.get('extra_options'),
'path': env_dict.get("path"),
'version': conf.get("version"),
'celery_app': conf.get("app"),
'has_cam': conf.get("celerycam"),
'enable_beat': conf.get("celerybeat"),
'cmd': cmd,
'pidfile': posixpath.normpath(posixpath.join(env_dict.get("path"), '..', 'tmp', 'celery-%s.pid' % site)),
}
celeryd_command_context.update(common_cmd_context)
celeryd_command_template_path = self.sites[site]['supervisor']['celeryd_command_template']
celeryd_command = render_template(
celeryd_command_template_path,
celeryd_command_context,
strip_newlines=True,
)
process_dict["%s_%s_celeryd" % (env_dict.get("user"), site)] = {
'command': celeryd_command,
'port': None,
'socket': None,
'type': 'celeryd',
'priority': 40,
'stopasgroup': 'true',
'killasgroup': 'true',
'stopwaitsecs': conf.get('stopwaitsecs', None),
}
if conf.get("celerycam"):
celerycam_command_context = {
'loglevel': conf.get("loglevel"),
'path': env_dict.get("path"),
'version': conf.get("version"),
'celery_app': conf.get("app"),
'cmd': cmd,
'frequency': conf.get('celerycam-frequency'),
}
celerycam_command_context.update(common_cmd_context)
celerycam_command_template_path = self.sites[site]['supervisor']['celerycam_command_template']
celerycam_command = render_template(
celerycam_command_template_path,
celerycam_command_context,
strip_newlines=True,
)
process_dict["%s_%s_celerycam" % (env_dict.get("user"), site)] = {
'command': celerycam_command,
'port': None,
'socket': None,
'type': 'celerycam',
'priority': 50,
}
if site_dict.get("redis").get("enabled"):
process_name = "%s_%s_redis" % (env_dict.get("user"), site)
redis_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", process_name + ".sock" )) # Asserts pony project layout
process_dict[process_name] = {
'command': "/usr/bin/redis-server %s" % posixpath.normpath(posixpath.join(env_dict.get('path'), '..', 'config', process_name + '.conf')),
'port': None,
'socket': redis_socket,
'type': 'redis',
'priority': 20,
}
if site_dict.get('processes'):
processes = site_dict.get('processes')
for process, command in processes.iteritems():
process_name = "%s_%s_process_%s" % (env_dict.get("user"), site, process)
process_dict[process_name] = {
'command': posixpath.join(env_dict.get("path"), command),
'port': None,
'socket': None,
'type': 'supervisor',
'priority': env_dict.get("priority", 200),
}
return process_dict
def processes(self, site, env_dict):
"""
Returns a dictionary of dictionaries each having the following keys:
* command
command to be run by supervisor
* port
port number,
* socket
path to unix socket
* type
gunicorn/memcached/celeryd
"""
process_dict = {}
site_dict = self.sites[site]
common_cmd_context = {
"django_cmd": site_dict.django['cmd'],
"django_args": " ".join(site_dict.get("django").get("args", [])),
}
gunicorn_cmd_context = {
"socket": posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_gunicorn.sock" % (env_dict.get("user"), site))), # Asserts pony project layout
"workers": site_dict.gunicorn['workers'],
}
gunicorn_cmd_context.update(common_cmd_context)
gunicorn_command_template_path = self.sites[site]['supervisor']['gunicorn_command_template']
gunicorn_command = render_template(gunicorn_command_template_path, gunicorn_cmd_context)
process_dict["%s_%s_gunicorn" % (env_dict.get("user"), site)] = {
'command': gunicorn_command,
'port': None,
'socket': gunicorn_cmd_context['socket'],
'type': 'gunicorn',
'priority': 100,
}
memcached_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_memcached.sock" % (env_dict.get("user"), site))) # Asserts pony project layout
process_dict["%s_%s_memcached" % (env_dict.get("user"), site)] = {
'command': "memcached -s %s" % memcached_socket,
'port': None,
'socket': memcached_socket,
'type': 'memcached',
'priority': 60,
}
if site_dict.get("celery").get("enabled"):
cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path"))
cmd += 'bin/celery'
celeryd_command_context = {
'concurrency': site_dict.get("celery").get("concurrency"),
'maxtasksperchild': site_dict.get("celery").get("maxtasksperchild"),
'loglevel': site_dict.get("celery").get("loglevel"),
'path': env_dict.get("path"),
'version': site_dict.get("celery").get("version"),
'celery_app': site_dict.get("celery").get("app"),
'has_cam': site_dict.get("celery").get("celerycam"),
'cmd': cmd,
}
celeryd_command_context.update(common_cmd_context)
celeryd_command_template_path = self.sites[site]['supervisor']['celeryd_command_template']
celeryd_command = render_template(celeryd_command_template_path, celeryd_command_context)
process_dict["%s_%s_celeryd" % (env_dict.get("user"), site)] = {
'command': celeryd_command,
'port': None,
'socket': None,
'type': 'celeryd',
'priority': 40,
}
if site_dict.get("celery").get("celerycam"):
celerycam_command_context = {
'loglevel': site_dict.get("celery").get("loglevel"),
'path': env_dict.get("path"),
'version': site_dict.get("celery").get("version"),
'celery_app': site_dict.get("celery").get("app"),
'cmd': cmd,
}
celerycam_command_context.update(common_cmd_context)
celerycam_command_template_path = self.sites[site]['supervisor']['celerycam_command_template']
celerycam_command = render_template(celerycam_command_template_path, celerycam_command_context)
process_dict["%s_%s_celerycam" % (env_dict.get("user"), site)] = {
'command': celerycam_command,
'port': None,
'socket': None,
'type': 'celerycam',
'priority': 50,
}
if site_dict.get("redis").get("enabled"):
process_name = "%s_%s_redis" % (env_dict.get("user"), site)
redis_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", process_name + ".sock" )) # Asserts pony project layout
process_dict[process_name] = {
'command': "/usr/bin/redis-server %s" % posixpath.normpath(posixpath.join(env_dict.get('path'), '..', 'config', process_name + '.conf')),
'port': None,
'socket': redis_socket,
'type': 'redis',
'priority': 20,
}
if site_dict.get('processes'):
processes = site_dict.get('processes')
for process, command in processes.iteritems():
process_name = "%s_%s_process_%s" % (env_dict.get("user"), site, process)
process_dict[process_name] = {
'command': posixpath.join(env_dict.get("path"), command),
'port': None,
'socket': None,
'type': 'supervisor',
'priority': env_dict.get("priority", 200),
}
return process_dict
