def update_config_file(): template = 'templates/supervisor/supervisor.conf' group_template = 'templates/supervisor/supervisor-group.conf' output = '' groups = {} for site, site_config in config.sites.items(): group_name = get_group_name(site, site_config) groups[group_name] = [] for process_name, process_dict in site_config.processes.items(): context_dict = copy(site_config) env_dict = { 'HOME': site_config.deployment['home'], 'PYTHONPATH': site_config.deployment['path'], } env_dict.update(site_config.environment) context_dict.update({ 'process_name': process_name, 'process_cmd': process_dict["command"], 'socket': process_dict["socket"], 'env': env_dict, 'priority': process_dict.get('priority', 200), }) output += render_template(template, context_dict) groups[group_name].append(process_name) output += render_template(group_template, {'groups': groups}) put(StringIO.StringIO(output), '%(path)s/../config/supervisor.conf' % config.sites["main"].deployment) update()
def update_config_file(dryrun=False, update_command=update, load_config=True): output = '' groups = {} for site, site_config in config.sites.items(): template_path = site_config['supervisor']['template'] group_template_path = site_config['supervisor']['group_template'] group_name = get_group_name(site, site_config) groups[group_name] = [] for process_name, process_dict in site_config.processes.items(): context_dict = copy(site_config) env_dict = { 'HOME': site_config.deployment['home'], 'USER': site_config.deployment['user'], 'PYTHONPATH': ":".join([ site_config.deployment['path'], posixpath.join(site_config.deployment['path'], site_config.get("django").get("base")+'/'), ]), } env_dict.update(site_config.environment) context_dict.update({ 'process_name': process_name, 'process_cmd': process_dict["command"], 'socket': process_dict["socket"], 'env': env_dict, 'priority': process_dict.get('priority', 200), 'autostart': 'True' if getattr(env, 'autostart', True) else 'False', 'killasgroup': process_dict.get('killasgroup', None), 'stopasgroup': process_dict.get('killasgroup', None), 'stopwaitsecs': process_dict.get('stopwaitsecs', None), }) output += render_template(template_path, context_dict) groups[group_name].append(process_name) output += render_template(group_template_path, {'groups': groups}) conf_path = posixpath.abspath(posixpath.join(config.sites["main"].deployment['path'], '..', 'config')) path = posixpath.join(conf_path, 'supervisor.conf') daemon_template_path = site_config['supervisor']['daemon_template'] supervisord_conf_path = posixpath.join(conf_path, 'supervisord.conf') supervisord_conf_output = render_template(daemon_template_path, copy(site_config)) if dryrun: print path + ':' print output print daemon_template_path + ':' print supervisord_conf_output else: safe_put(StringIO.StringIO(output), path) safe_put(StringIO.StringIO(supervisord_conf_output), supervisord_conf_path) if load_config: update_command()
def update_config_file(dryrun=False): for site, site_config in list(config.sites.items()): redis_processes = [(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"] template_path = site_config['redis']['template'] print(redis_processes) for process_name, process in redis_processes: working_directoy = posixpath.normpath( posixpath.join(env.path, '..', 'data', 'redis', process_name)) log_directory = posixpath.normpath( posixpath.join(site_config['deployment']['logdir'], 'log', 'redis')) run('mkdir -p ' + working_directoy) run('mkdir -p ' + log_directory) context_dict = site_config context_dict.update({ 'site': site, 'working_directory': working_directoy, 'log_directory': log_directory, 'process_name': process_name, 'socket': process['socket'], }) path = posixpath.abspath( posixpath.join(site_config['deployment']['path'], '..', 'config', process_name + '.conf')) output = render_template(template_path, context_dict) if dryrun: print((path + ":")) print(output) else: put(io.StringIO(output), path)
def update_config_file(dryrun=False): for site, site_config in config.sites.items(): redis_processes = [ (x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis" ] template_path = site_config["redis"]["template"] print redis_processes for process_name, process in redis_processes: working_directoy = posixpath.normpath(posixpath.join(env.path, "..", "data", "redis", process_name)) log_directory = posixpath.normpath(posixpath.join(env.path, "..", "log", "redis")) run("mkdir -p " + working_directoy) run("mkdir -p " + log_directory) context_dict = site_config context_dict.update( { "site": site, "working_directory": working_directoy, "log_directory": log_directory, "process_name": process_name, "socket": process["socket"], } ) path = posixpath.abspath( posixpath.join(site_config["deployment"]["path"], "..", "config", process_name + ".conf") ) output = render_template(template_path, context_dict) if dryrun: print path + ":" print output else: put(StringIO.StringIO(output), path)
def update_config_file(dryrun=False): for site, site_config in config.sites.items(): redis_processes = [(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"] template_path = site_config['redis']['template'] print redis_processes for process_name, process in redis_processes: working_directoy = posixpath.normpath(posixpath.join(env.path, '..', 'data', 'redis', process_name)) log_directory = posixpath.normpath(posixpath.join(site_config['deployment']['logdir'], 'log', 'redis')) run('mkdir -p ' + working_directoy) run('mkdir -p ' + log_directory) context_dict = site_config context_dict.update({ 'site': site, 'working_directory': working_directoy, 'log_directory': log_directory, 'process_name': process_name, 'socket': process['socket'], }) path = posixpath.abspath(posixpath.join(site_config['deployment']['path'], '..', 'config', process_name + '.conf')) output = render_template(template_path, context_dict) if dryrun: print path + ":" print output else: put(StringIO.StringIO(output), path)
def update_config_file(dryrun=False, update_command=update): output = "" groups = {} for site, site_config in config.sites.items(): template_path = site_config["supervisor"]["template"] group_template_path = site_config["supervisor"]["group_template"] group_name = get_group_name(site, site_config) groups[group_name] = [] for process_name, process_dict in site_config.processes.items(): context_dict = copy(site_config) env_dict = { "HOME": site_config.deployment["home"], "USER": site_config.deployment["user"], "PYTHONPATH": ":".join( [ site_config.deployment["path"], posixpath.join(site_config.deployment["path"], site_config.get("django").get("base") + "/"), ] ), } env_dict.update(site_config.environment) context_dict.update( { "process_name": process_name, "process_cmd": process_dict["command"], "socket": process_dict["socket"], "env": env_dict, "priority": process_dict.get("priority", 200), "autostart": "True" if getattr(env, "autostart", True) else "False", "killasgroup": process_dict.get("killasgroup", None), "stopasgroup": process_dict.get("killasgroup", None), "stopwaitsecs": process_dict.get("stopwaitsecs", None), } ) output += render_template(template_path, context_dict) groups[group_name].append(process_name) output += render_template(group_template_path, {"groups": groups}) path = posixpath.abspath(posixpath.join(config.sites["main"].deployment["path"], "..", "config", "supervisor.conf")) if dryrun: print path + ":" print output else: put(StringIO.StringIO(output), path) update_command()
def update_config_file(): template = 'templates/supervisor/supervisor.conf' output = '' for site, site_config in config.sites.items(): for process_name, process_dict in site_config.processes.items(): context_dict = site_config context_dict.update({'process_name': process_name, 'process_cmd': process_dict["command"], 'socket': process_dict["socket"]}) output += render_template(template, context_dict) put(StringIO.StringIO(output), '%(path)s/../config/supervisor.conf' % config.sites["main"].deployment) update()
def update_config_file(): output = "" template_name = 'templates/nginx/nginx.conf' for site, site_config in config.sites.items(): context_dict = site_config context_dict.update({ 'domains': " ".join(site_config.deployment.get("domains")[site]), 'www_processes': [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"], }) output += render_template(template_name, context_dict) put(StringIO.StringIO(output), '%(path)s/../config/nginx.conf' % env) reload_nginx()
def update_config_file(): template = 'templates/supervisor/supervisor.conf' output = '' for site, site_config in config.sites.items(): for process_name, process_dict in site_config.processes.items(): context_dict = site_config context_dict.update({ 'process_name': process_name, 'process_cmd': process_dict["command"], 'socket': process_dict["socket"] }) output += render_template(template, context_dict) put(StringIO.StringIO(output), '%(path)s/../config/supervisor.conf' % config.sites["main"].deployment) update()
def update_config_file(dryrun=False): output = "" for site, site_config in config.sites.items(): context_dict = site_config context_dict.update({ 'domains': " ".join(site_config.deployment.get("domains")[site]), 'www_processes': [site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn"], }) template_path = context_dict['nginx']['template'] output += render_template(template_path, context_dict) path = posixpath.abspath(posixpath.join(env.path, '..', 'config', 'nginx.conf')) if dryrun: print path + ':' print output else: put(StringIO.StringIO(output), path) reload_nginx()
def update_config_file(): template_name = 'templates/redis/redis.conf' for site, site_config in config.sites.items(): redis_processes = [(x, site_config.processes[x]) for x in site_config.processes if site_config.processes[x]["type"] == "redis"] print redis_processes for process_name, process in redis_processes: working_directoy = posixpath.normpath(posixpath.join(env.path, '..', 'data', 'redis', process_name)) log_directory = posixpath.normpath(posixpath.join(env.path, '..', 'log', 'redis')) run('mkdir -p ' + working_directoy) run('mkdir -p ' + log_directory) context_dict = site_config context_dict.update({ 'site': site, 'working_directory': working_directoy, 'log_directory': log_directory, 'process_name': process_name, 'socket': process['socket'], }) output = render_template(template_name, context_dict) put(StringIO.StringIO(output), posixpath.join(site_config['deployment']['path'], '..', 'config', process_name + '.conf'))
def update_config_file(dryrun=False): output = "" for site, site_config in config.sites.items(): context_dict = site_config context_dict.update({ 'domains': " ".join(site_config.deployment.get("domains")[site]), 'www_processes': [ site_config.processes[x] for x in site_config.processes if site_config.processes[x]["type"] == "gunicorn" ], }) template_path = context_dict['nginx']['template'] output += render_template(template_path, context_dict) path = posixpath.abspath( posixpath.join(env.path, '..', 'config', 'nginx.conf')) if dryrun: print path + ':' print output else: put(StringIO.StringIO(output), path) reload_nginx()
def processes(self, site, env_dict): """ Returns a dictionary of dictionaries each having the following keys: * command command to be run by supervisor * port port number, * socket path to unix socket * type gunicorn/memcached/celeryd """ process_dict = {} site_dict = self.sites[site] common_cmd_context = { "django_cmd": site_dict.django['cmd'], "django_args": " ".join(site_dict.get("django").get("args", [])), } socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_gunicorn.sock" % (env_dict.get("user"), site))) # Asserts pony project layout if site_dict.gunicorn['bind']: bind = site_dict.gunicorn['bind'] else: bind = 'unix:{}'.format(socket) cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path")) cmd += 'bin/gunicorn' gunicorn_cmd_context = { 'cmd': cmd, "socket": socket, "bind": bind, "workers": site_dict.gunicorn['workers'], "maxrequests": site_dict.gunicorn['maxrequests'], "timeout": site_dict.gunicorn['timeout'], "version": site_dict.gunicorn['version'], } gunicorn_cmd_context.update(common_cmd_context) gunicorn_command_template_path = self.sites[site]['supervisor']['gunicorn_command_template'] gunicorn_command = render_template( gunicorn_command_template_path, gunicorn_cmd_context, strip_newlines=True, ) process_dict["%s_%s_gunicorn" % (env_dict.get("user"), site)] = { 'command': gunicorn_command, 'port': None, 'socket': gunicorn_cmd_context['socket'], 'type': 'gunicorn', 'priority': 100, } custom_processes = env_dict.get("custom_processes") or [] for process in custom_processes: process_config = site_dict[process] if not process_config.get("enabled"): continue custom_command = process_config['command'] if process_config.get('django'): custom_command = '%s %s' % (site_dict.django['cmd'], custom_command) process_name = "%s_%s_%s" % (env_dict.get("user"), site, process) process_dict[process_name] = { 'command': custom_command, 'type': 'custom', 'priority': 100, 'port': None, 'socket': None, } if site_dict.get("memcached").get("enabled"): memcached_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_memcached.sock" % (env_dict.get("user"), site))) # Asserts pony project layout process_dict["%s_%s_memcached" % (env_dict.get("user"), site)] = { 'command': "memcached -s %s -m %d" % (memcached_socket, int(site_dict.get("memcached").get("size"))), 'port': None, 'socket': memcached_socket, 'type': 'memcached', 'priority': 60, } if site_dict.get("celery").get("enabled"): conf = site_dict.get("celery") cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path")) cmd += 'bin/celery' celeryd_command_context = { 'concurrency': conf.get("concurrency"), 'maxtasksperchild': conf.get("maxtasksperchild"), 'loglevel': conf.get("loglevel"), 'extra_options': conf.get('extra_options'), 'path': env_dict.get("path"), 'version': conf.get("version"), 'celery_app': conf.get("app"), 'has_cam': conf.get("celerycam"), 'enable_beat': conf.get("celerybeat"), 'cmd': cmd, 'pidfile': posixpath.normpath(posixpath.join(env_dict.get("path"), '..', 'tmp', 'celery-%s.pid' % site)), } celeryd_command_context.update(common_cmd_context) celeryd_command_template_path = self.sites[site]['supervisor']['celeryd_command_template'] celeryd_command = render_template( celeryd_command_template_path, celeryd_command_context, strip_newlines=True, ) process_dict["%s_%s_celeryd" % (env_dict.get("user"), site)] = { 'command': celeryd_command, 'port': None, 'socket': None, 'type': 'celeryd', 'priority': 40, 'stopasgroup': 'true', 'killasgroup': 'true', 'stopwaitsecs': conf.get('stopwaitsecs', None), } if conf.get("celerycam"): celerycam_command_context = { 'loglevel': conf.get("loglevel"), 'path': env_dict.get("path"), 'version': conf.get("version"), 'celery_app': conf.get("app"), 'cmd': cmd, 'frequency': conf.get('celerycam-frequency'), } celerycam_command_context.update(common_cmd_context) celerycam_command_template_path = self.sites[site]['supervisor']['celerycam_command_template'] celerycam_command = render_template( celerycam_command_template_path, celerycam_command_context, strip_newlines=True, ) process_dict["%s_%s_celerycam" % (env_dict.get("user"), site)] = { 'command': celerycam_command, 'port': None, 'socket': None, 'type': 'celerycam', 'priority': 50, } if site_dict.get("redis").get("enabled"): process_name = "%s_%s_redis" % (env_dict.get("user"), site) redis_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", process_name + ".sock" )) # Asserts pony project layout process_dict[process_name] = { 'command': "/usr/bin/redis-server %s" % posixpath.normpath(posixpath.join(env_dict.get('path'), '..', 'config', process_name + '.conf')), 'port': None, 'socket': redis_socket, 'type': 'redis', 'priority': 20, } if site_dict.get('processes'): processes = site_dict.get('processes') for process, command in processes.iteritems(): process_name = "%s_%s_process_%s" % (env_dict.get("user"), site, process) process_dict[process_name] = { 'command': posixpath.join(env_dict.get("path"), command), 'port': None, 'socket': None, 'type': 'supervisor', 'priority': env_dict.get("priority", 200), } return process_dict
def processes(self, site, env_dict): """ Returns a dictionary of dictionaries each having the following keys: * command command to be run by supervisor * port port number, * socket path to unix socket * type gunicorn/memcached/celeryd """ process_dict = {} site_dict = self.sites[site] common_cmd_context = { "django_cmd": site_dict.django['cmd'], "django_args": " ".join(site_dict.get("django").get("args", [])), } gunicorn_cmd_context = { "socket": posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_gunicorn.sock" % (env_dict.get("user"), site))), # Asserts pony project layout "workers": site_dict.gunicorn['workers'], } gunicorn_cmd_context.update(common_cmd_context) gunicorn_command_template_path = self.sites[site]['supervisor']['gunicorn_command_template'] gunicorn_command = render_template(gunicorn_command_template_path, gunicorn_cmd_context) process_dict["%s_%s_gunicorn" % (env_dict.get("user"), site)] = { 'command': gunicorn_command, 'port': None, 'socket': gunicorn_cmd_context['socket'], 'type': 'gunicorn', 'priority': 100, } memcached_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", "%s_%s_memcached.sock" % (env_dict.get("user"), site))) # Asserts pony project layout process_dict["%s_%s_memcached" % (env_dict.get("user"), site)] = { 'command': "memcached -s %s" % memcached_socket, 'port': None, 'socket': memcached_socket, 'type': 'memcached', 'priority': 60, } if site_dict.get("celery").get("enabled"): cmd = env_dict.get("path") if not site_dict.get("newrelic").get("enabled") else '%sbin/newrelic-admin run-program %s' % (env_dict.get("path"), env_dict.get("path")) cmd += 'bin/celery' celeryd_command_context = { 'concurrency': site_dict.get("celery").get("concurrency"), 'maxtasksperchild': site_dict.get("celery").get("maxtasksperchild"), 'loglevel': site_dict.get("celery").get("loglevel"), 'path': env_dict.get("path"), 'version': site_dict.get("celery").get("version"), 'celery_app': site_dict.get("celery").get("app"), 'has_cam': site_dict.get("celery").get("celerycam"), 'cmd': cmd, } celeryd_command_context.update(common_cmd_context) celeryd_command_template_path = self.sites[site]['supervisor']['celeryd_command_template'] celeryd_command = render_template(celeryd_command_template_path, celeryd_command_context) process_dict["%s_%s_celeryd" % (env_dict.get("user"), site)] = { 'command': celeryd_command, 'port': None, 'socket': None, 'type': 'celeryd', 'priority': 40, } if site_dict.get("celery").get("celerycam"): celerycam_command_context = { 'loglevel': site_dict.get("celery").get("loglevel"), 'path': env_dict.get("path"), 'version': site_dict.get("celery").get("version"), 'celery_app': site_dict.get("celery").get("app"), 'cmd': cmd, } celerycam_command_context.update(common_cmd_context) celerycam_command_template_path = self.sites[site]['supervisor']['celerycam_command_template'] celerycam_command = render_template(celerycam_command_template_path, celerycam_command_context) process_dict["%s_%s_celerycam" % (env_dict.get("user"), site)] = { 'command': celerycam_command, 'port': None, 'socket': None, 'type': 'celerycam', 'priority': 50, } if site_dict.get("redis").get("enabled"): process_name = "%s_%s_redis" % (env_dict.get("user"), site) redis_socket = posixpath.normpath(posixpath.join(env_dict.get("path"), "..", "tmp", process_name + ".sock" )) # Asserts pony project layout process_dict[process_name] = { 'command': "/usr/bin/redis-server %s" % posixpath.normpath(posixpath.join(env_dict.get('path'), '..', 'config', process_name + '.conf')), 'port': None, 'socket': redis_socket, 'type': 'redis', 'priority': 20, } if site_dict.get('processes'): processes = site_dict.get('processes') for process, command in processes.iteritems(): process_name = "%s_%s_process_%s" % (env_dict.get("user"), site, process) process_dict[process_name] = { 'command': posixpath.join(env_dict.get("path"), command), 'port': None, 'socket': None, 'type': 'supervisor', 'priority': env_dict.get("priority", 200), } return process_dict