def do_add():
cfg_dir = os.path.dirname(self.site_config_path())
if not os.path.exists(cfg_dir):
run_cmd("mkdir -p %s" % cfg_dir, _log,
"Creating Varnish config dir")
with open(self.site_config_path(), "w") as f:
json.dump(config, f, indent=2)
def __init__(self):
self.rollbacks = []
self.webserver_reload = False
self.varnish_reload_cmd = None
self.curr_idx = ConfigTransaction.file_idx
ConfigTransaction.file_idx += 1
varnish_dir = "/etc/varnish"
if not os.path.exists(varnish_dir):
varnish_dir = ""
etc_dir = PlatformWebServer().etc_dir()
self.run(
lambda: run_cmd(
"rm -Rf /tmp/revsw-apache-config.%d.tar && tar cf "
"/tmp/revsw-apache-config.%d.tar /opt/revsw-config/apache "
"/opt/revsw-config/varnish %s/sites-enabled %s/sites-available %s --exclude=%s"
% (self.curr_idx, self.curr_idx, etc_dir, etc_dir, varnish_dir,
ConfigTransaction.backup_file), _log,
"Backing up existing config"),
lambda: run_cmd(
"rm -Rf /opt/revsw-config/apache /opt/revsw-config/varnish %s/"
"sites-enabled %s/sites-available %s && tar -C / -xf /tmp/revsw-apache-config.%d.tar"
% (etc_dir, etc_dir, varnish_dir, self.curr_idx
), _log, "Restoring previous config"))
def reload_or_start():
reload_cmd = "reload"
try:
run_cmd("service revsw-nginx status", _log,
"Checking if Nginx is running", True)
except OSError:
reload_cmd = "start"
# Is our config valid ?
# Can't use run_cmd because we need the stderr output to determine which site(s) have caused
# failures.
_log.LOGI("Checking Nginx config")
child = subprocess.Popen("service revsw-nginx configtest",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = child.communicate()
if child.returncode != 0:
for line in stderr.split("\n"):
_log.LOGE(line)
raise ConfigException("Nginx config check failed",
NginxConfig.get_error_domains(stderr))
run_cmd("service revsw-nginx %s" % reload_cmd, _log,
"%sing Nginx" % reload_cmd.capitalize(), True)
def do_write():
sites = []
_log.LOGI("Loading input vars from JSON")
# TODO: we need to replace path to variable outside this class
fnames = sorted([
"%ssites/%s.json" %
(script_configs.VARNISH_PATH_CONFIG, _(dom))
for dom in NginxConfig.get_all_active_domains()
])
_log.LOGI(" -> files: ", fnames)
for fname in fnames:
with open(fname) as f:
site = json.load(
f, object_pairs_hook=dict_raise_on_duplicates)
sites.append(site)
input_vars = {"sites": sites}
# _log.LOGD(json.dumps(input_vars, indent=2))
self._lazy_load_template(search_path)
jsch.validate(input_vars,
self.input_vars_schema,
format_checker=jsch.FormatChecker())
cfg = self.varn_template.render(input_vars)
# _log.LOGI("Generated VCL:", cfg)
cfg = cfg.replace('\r', '\n')
cfg = cfg.replace('\n\n', '\n')
cfg = cfg.replace('\n\n', '\n')
conf_file_name = os.path.join(script_configs.VARNISH_PATH,
"revsw.vcl")
with open(conf_file_name + ".tmp", "w") as f:
f.write(cfg)
# # re-formatting Varnish config file:
# run_cmd("cat %(INPUT)s.tmp | %(CONFIG_PATH)sbin/conf_files_formatter.sh >
# %(OUTPUT)s && mv %(INPUT)s.tmp %(TMP_PATH)s" % \
# {
# "INPUT": conf_file_name, "OUTPUT": conf_file_name,
# "TMP_PATH": script_configs.TMP_PATH, "CONFIG_PATH": script_configs.CONFIG_PATH
# },
# _log, "re-formatting %s file" % conf_file_name
# )
# # .
# TODO: remove it after test
run_cmd(
"cp %(INPUT)s.tmp %(OUTPUT)s && mv %(INPUT)s.tmp %(TMP_PATH)s"
% {
"INPUT": conf_file_name,
"OUTPUT": conf_file_name,
"TMP_PATH": script_configs.TMP_PATH,
"CONFIG_PATH": script_configs.CONFIG_PATH
}, _log, "re-formatting %s file" % conf_file_name)
def _write_template_files(files, output_dir):
run_cmd("mkdir -p %s" % output_dir, _log, silent=True)
for fname, content in files.iteritems():
dirname = os.path.join(output_dir, os.path.dirname(fname))
basename = os.path.basename(fname)
run_cmd("mkdir -p %s" % dirname, _log, silent=True)
with open(os.path.join(dirname, basename), "w") as f:
f.write(content)
def finalize(self):
"""Finishes the transaction by reloading either varnish or webserver
if either is nessesary. Also backs up previous config in
"/var/cache/revsw-apache-old-config.tar" of local server.
Raises:
ConfigException: If varnish fails to reload or start it will raise
this exception.
"""
if self.webserver_reload:
reload_func = NginxConfig.reload_or_start
self.run(reload_func)
# In case the Varnish reload fails, reload Apache again after the
# old config has been restored
self.rollbacks.insert(0, reload_func)
if self.varnish_reload_cmd:
if VarnishConfig.varnish_is_installed():
v_reload_cmd = self.varnish_reload_cmd
try:
run_cmd("service revsw-varnish4 status", _log,
"Checking if Varnish is running", True)
except OSError:
v_reload_cmd = "start"
VarnishConfig(transaction=self).write_config_file()
def reload_varnish():
# Can't use run_cmd because we need the stderr output to determine which site(s) have caused
# failures.
child = subprocess.Popen("service revsw-varnish4 %s" %
v_reload_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = child.communicate()
_log.LOGI("%sing Varnish" % v_reload_cmd.capitalize())
if child.returncode != 0:
for line in stderr.split("\n"):
_log.LOGE(line)
raise ConfigException(
"Varnish %s failed" % v_reload_cmd,
VarnishConfig.get_error_domains(stderr))
self.run(reload_varnish)
else:
_log.LOGI("Varnish is not installed; not %sing it" %
self.varnish_reload_cmd)
# Finally, save the previous config for reference
run_cmd(
"mv -f /tmp/revsw-apache-config.%d.tar %s" %
(self.curr_idx, ConfigTransaction.backup_file), _log,
"Saving previous config to '%s'" % ConfigTransaction.backup_file)
def fixup_file(fname):
fdir = os.path.join(jinja_config_webserver_dir(self.site_name),
"certs")
fpath = os.path.join(fdir, fname)
if not os.path.exists(fpath):
run_cmd(
"mkdir -p %s && ln -f -s %s %s" %
(fdir,
os.path.join(jinja_config_webserver_base_dir(),
"generic-site", "certs", fname), fpath),
_log, "Creating default '%s'" % fname)
def remove_site(self):
"""Removes current working site pointed to by self.site_name by removing all
Nginx .conf files and jinja templates if they exist
"""
self.transaction.run(lambda: run_cmd(
"rm -f %ssites-enabled/%s.conf" %
(script_configs.NGINX_PATH, self.site_name
), _log, "Disabling site '%s' if it exists" % self.site_name))
self.transaction.run(lambda: run_cmd(
"rm -f %ssites-available/%s.conf" %
(script_configs.NGINX_PATH, self.site_name
), _log, "Removing site '%s' if it exists" % self.site_name))
self.transaction.run(lambda: run_cmd(
"rm -Rf %s" % jinja_config_webserver_dir(self.site_name), _log,
"Removing site '%s' templates, if they exist" % self.site_name))
def __init__(self):
global _g_webserver_name
if not _g_webserver_name:
for pkg, name in (("revsw-nginx-full", "NGINX"),
("revsw-nginx-naxsi", "NGINX")):
try:
run_cmd("dpkg-query -s %s" % pkg, _log, silent=True)
if _g_webserver_name:
raise RuntimeError(
"Both Nginx versions are installed; please check your configuration"
)
_g_webserver_name = name
except OSError:
pass
if not _g_webserver_name:
raise RuntimeError(
"Neither Nginx Full nor Nginx Naxsi are installed; please check your configuration"
)
self._name = _g_webserver_name
def write_certs():
certs_dir = os.path.join(
jinja_config_webserver_dir(self.site_name), "certs")
run_cmd("rm -Rf %s" % certs_dir, _log,
"Removing directory '%s' if it exists" % certs_dir)
run_cmd("mkdir -p %s" % certs_dir, _log,
"Creating directory '%s" % certs_dir)
_log.LOGI("Creating certificate files")
base64_string_gzip_to_file(cert, "%s/server.crt" % certs_dir)
base64_string_gzip_to_file(key, "%s/server.key" % certs_dir)
base64_string_gzip_to_file(ca_bundle,
"%s/ca-bundle.crt" % certs_dir)
run_cmd(
"cat %s/server.crt %s/ca-bundle.crt > %s/server-chained.crt" %
(certs_dir, certs_dir, certs_dir), _log,
"Creating server-chained.crt")
run_cmd(
"chmod 0400 %s/server.key %s/server-chained.crt" %
(certs_dir, certs_dir), _log, "Fixing permissions")
def do_write():
search_dirs = [jinja_config_webserver_dir(self.site_name)]
template_file_no_ext = self._template_file_no_ext()
_log.LOGD("Loading template %s" % template_file_no_ext)
with open("%s.jinja" % template_file_no_ext) as f:
template_str = f.read()
_log.LOGD("Loading vars schema %s" % template_file_no_ext)
with open("%s.vars.schema" % template_file_no_ext) as f:
schema = json.load(f,
object_pairs_hook=dict_raise_on_duplicates)
_log.LOGD("Validating input vars from JSON")
jsch.validate(input_vars,
schema,
format_checker=jsch.FormatChecker())
env = SandboxedEnvironment(
line_statement_prefix=None,
trim_blocks=True,
lstrip_blocks=True,
loader=jinja2.FileSystemLoader(search_dirs),
undefined=StrictUndefined,
extensions=["jinja2.ext.do"])
(hostname_short, hostname_full) = _get_hostname_short_and_full()
env.filters["flatten_to_set"] = flatten_to_set
env.filters["parse_url"] = parse_url
env.filters["dns_query"] = dns_query
env.filters["underscore_url"] = underscore_url
env.filters["is_ipv4"] = is_ipv4
env.filters["wildcard_to_regex"] = wildcard_to_regex
env.filters[
"extract_custom_webserver_code"] = extract_custom_webserver_code
env.filters["netmask_bits"] = netmask_bits
env.globals["global_var_get"] = global_var_get
env.globals["global_var_set"] = global_var_set
env.globals["GLOBAL_SITE_NAME"] = self.site_name
env.globals["HOSTNAME_FULL"] = hostname_full
env.globals["HOSTNAME_SHORT"] = hostname_short
env.globals["DNS_SERVERS"] = dns_servers()
env.globals["bypass_location_root"] = False
template = env.from_string(template_str)
cfg = template.render(input_vars)
cfg = cfg.replace('\r', '\n')
cfg = cfg.replace('\n\n', '\n')
cfg = cfg.replace('\n\n', '\n')
# TODO: we need to replace path to NGINX to variable outside this
# class
conf_file_name = "%ssites-available/%s.conf" % (
script_configs.NGINX_PATH, self.site_name)
with open(conf_file_name + ".tmp", "w") as f:
f.write(cfg)
# re-formatting Nginx config file:
run_cmd(
"cat %(INPUT)s.tmp | %(CONFIG_PATH)sbin/conf_files_formatter.sh > "
"%(OUTPUT)s && mv %(INPUT)s.tmp /tmp/" % {
"INPUT": conf_file_name,
"OUTPUT": conf_file_name,
"CONFIG_PATH": script_configs.CONFIG_PATH
}, _log, "re-formatting %s file" % conf_file_name)
# .
_log.LOGD("Generated Nginx config file")
# Make sure the site has at least the default certs
self._fixup_certs()
run_cmd(
"cd %(NGINX_PATH)ssites-enabled && ln -sf %(NGINX_PATH)ssites-available/%(SITE_NAME)s.conf"
% {
"NGINX_PATH": script_configs.NGINX_PATH,
"SITE_NAME": self.site_name
}, _log, "Enabling site '%s' if necessary" % self.site_name)
_log.LOGD("Saving input vars")
with open("%s.json" % template_file_no_ext, "w") as f:
json.dump(input_vars, f, indent=2)
def disable_all_sites(self):
"""Disables all sites by removing all sites from /etc/nginx/sites-enabled"""
self.transaction.run(lambda: run_cmd(
"rm -f /etc/nginx/sites-enabled/*", _log, "Disabling all sites"))
def configure_all(config):
"""Run all configuration actions described in the config provided. Will only
run needed utilities if they are changed.
Args:
config (dict): Configuration parameters.
"""
_log.LOGD(u"Input CONFIG is: ", json.dumps(config))
if "version" not in config:
raise AttributeError("No version info in configuration")
if config["version"] != script_configs.API_VERSION:
raise AttributeError(
"Incompatible version %d in configuration; expected %d" %
(config["version"], script_configs.API_VERSION))
transaction = ConfigTransaction()
for command in config["commands"]:
action = _check_and_get_attr(command, "type")
if not action:
raise AttributeError("Action not specified; ignoring")
if action == "flush" or action == "varnish_template" or action == "mlogc_template":
site = "*"
else:
site = _check_and_get_attr(command, "site_name")
if not site:
raise AttributeError("Site not specified; ignoring")
acfg = NginxConfig(site, transaction)
vcfg = VarnishConfig(site, transaction)
_log.LOGI("Got request for site '%s', action '%s'" % (site, action))
if action == "flush":
_log.LOGD("Removing all sites")
acfg.disable_all_sites()
# If site == '*', remove_site() will remove all sites
vcfg.remove_site()
elif action == "varnish_template":
_log.LOGD("Writing Varnish template")
vcfg.write_template_files(_check_and_get_attr(
command, "templates"))
elif action == "delete":
_domain_name = _check_and_get_attr(command, "domain_name")
transaction.run(lambda: run_cmd(
"rm -f /opt/revsw-config/policy/ui-config-%s.json" %
_domain_name, _log, "Removing policy '%s' if it exists" %
_domain_name))
acfg.remove_site()
vcfg.remove_site()
_log.LOGD("Removing site '%s'" % _domain_name)
elif action == "batch":
_log.LOGD("Batch configuring site '%s'" % site)
templates = command.get("templates")
if templates:
if "nginx" not in templates:
raise AttributeError("Config templates don't presented")
real_templates = templates["nginx"]
_log.LOGD("Writing Jinja templates")
acfg.write_template_files(real_templates)
cfg_vars = _check_and_get_attr(command, "config_vars")
acfg.configure_site(cfg_vars)
varnish_config_vars = command.get("varnish_config_vars")
if varnish_config_vars:
vcfg.config_site(varnish_config_vars)
transaction.varnish_reload_cmd = None
transaction.webserver_reload = False
elif action == "config" or action == "force":
_log.LOGD("Configuring site '%s'" % site)
templates = command.get("templates")
if templates:
if "nginx" not in templates:
raise AttributeError("Config templates don't presented")
real_templates = templates["nginx"]
_log.LOGD("Writing Jinja templates")
acfg.write_template_files(real_templates)
cfg_vars = _check_and_get_attr(command, "config_vars")
acfg.configure_site(cfg_vars)
varnish_changed_vars = config["varnish_changed"]
varnish_config_vars = command.get("varnish_config_vars")
if varnish_changed_vars:
if varnish_config_vars:
vcfg.config_site(varnish_config_vars)
else: # Varnish not needed for site, remove if present
vcfg.remove_site()
else:
if varnish_config_vars:
vcfg.config_site(varnish_config_vars)
transaction.varnish_reload_cmd = None
_log.LOGD("No changes in Varnish configuration")
config_changed_vars = config["config_changed"]
if config_changed_vars:
transaction.webserver_reload = True
else:
transaction.webserver_reload = False
if action == "force": # Forces nginx and varnish to reload
transaction.webserver_reload = True
transaction.schedule_varnish_reload()
_log.LOGD("Config changed: ", transaction.webserver_reload)
elif action == "certs":
_log.LOGD("Configuring site '%s' certificates" % site)
certs = _check_and_get_attr(command, "certs")
acfg.write_certs(certs["crt"], certs["key"], certs["ca-bundle"])
else:
raise AttributeError("Invalid action '%s'" % action)
# Reload the configs and save the old config
_log.LOGD("Webserver reload status: '%s'" % transaction.webserver_reload)
_log.LOGD("Varnish reload status: '%s'" % transaction.varnish_reload_cmd)
transaction.finalize()
return transaction
def remove_site(self):
self.transaction.run(
lambda: run_cmd("rm -f %s" % self.site_config_path(), _log,
"Removing Varnish config"))
def varnish_is_installed():
try:
run_cmd("which varnishd", _log, silent=True)
except OSError:
return False
return True