def create_slave_conf(master_ip):
""" Method in charge to build configurations files of a
slave Redis host (redis.conf + sentinel.conf)
:param master_ip: IP Address or hostname of Master host
:return:
"""
parameters = {
'is_redis_master': False,
'master_ip': master_ip,
}
# Creating Redis.conf
tpl, path = tpl_utils.get_template('redis')
conf = tpl.render(conf=parameters)
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
# Creating Sentinel conf
tpl, path = tpl_utils.get_template('sentinel')
conf = tpl.render(conf=parameters)
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
def create_master_conf():
""" Method in charge to build configurations files of a
master Redis host (redis.conf + sentinel.conf)
:return:
"""
from vulture_toolkit.network.net_utils import get_hostname
master_ip = get_hostname()
parameters = {
'is_redis_master': True,
'master_ip': master_ip,
}
tpl, path = tpl_utils.get_template('redis')
conf = tpl.render(conf=parameters)
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
# Creating Sentinel conf
tpl, path = tpl_utils.get_template('sentinel')
conf = tpl.render(conf=parameters)
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
def conf_has_changed(self,
parameters,
service_settings=None,
conf_django=None):
tpl, path = tpl_utils.get_template(self.service_mail_name)
conf = tpl.render()
try:
with open(path, 'r') as f:
orig_conf = f.read()
if orig_conf != conf:
return conf
except Exception as e:
logger.error("Unable to check if conf has changed for {}: ".format(
self.service_mail_name, str(e)))
logger.exception(e)
if "No such file" in str(e):
raise ServiceNoConfigError(str(e))
raise ServiceConfigError(
"Cannot open '{}' for service '{}' : {}".format(
path, self.service_mail_name, str(e)))
main_changed = super(SMTP,
self).conf_has_changed(parameters,
service_settings,
conf_django)
if "No such file" in main_changed:
raise ServiceNoConfigError(main_changed)
elif "Unable to check if conf has changed" in main_changed:
raise ServiceConfigError(main_changed)
def process(self):
defender_dir_path = settings.CONF_DIR + "defender/"
if not os.path.exists(defender_dir_path):
os.makedirs(defender_dir_path)
core_rules_dst_path = defender_dir_path + "core.rules"
tpl, path = tpl_utils.get_template('defender_core_rules')
defender_core_rules = tpl.render(
conf={
'sql_warn': 1,
'sql_err': 2,
'sql_crit': 4,
'rfi_warn': 1,
'rfi_err': 2,
'rfi_crit': 4,
'traversal_warn': 1,
'traversal_err': 2,
'traversal_crit': 4,
'xss_warn': 1,
'xss_err': 2,
'xss_crit': 4,
'evade_warn': 1,
'evade_err': 2,
'evade_crit': 4,
'upload_warn': 1,
'upload_err': 2,
'upload_crit': 4
})
with open(core_rules_dst_path, 'w') as f:
f.write(defender_core_rules)
print("ModDefender core rules generated")
def conf_has_changed(self,
parameters,
service_settings=None,
conf_django=None):
""" Test if configuration into system configuration file is same that
configuration in database (from parameters var)
:param parameters: A dict containing configuration parameters
:param service_settings: A dict containing configuration parameters, sent to tempale.render as 'conf_service'
:param conf_django: A dict containing configuration parameters, sent to template.render as 'conf_django'
"""
tpl, path = tpl_utils.get_template(self.service_name)
conf = tpl.render(conf=parameters,
conf_service=service_settings,
conf_django=conf_django)
try:
with open(path, 'r') as f:
orig_conf = f.read()
if orig_conf != conf:
return conf
return ""
except Exception as e:
logger.error("Unable to check if conf has changed for {}: ".format(
self.service_name, str(e)))
logger.exception(e)
return "Unable to check if conf has changed : {}".format(str(e))
def write_configuration(self, parameters, service_settings=None):
for file in ["strongswan", "ipsec", "ipsec-secrets"]:
tpl, path = tpl_utils.get_template(file)
conf = tpl.render({'conf': parameters})
logger.info("[IPSEC] - Writing configuration file {}".format(file))
write_in_file(path, conf)
def write_configuration(self,
parameters,
service_settings=None,
text_config=None):
# ### Write the second template (mailer.conf)
tpl, path = tpl_utils.get_template(self.service_mail_name)
conf = tpl.render()
logger.info("{} configuration has changed, writing new "
"one".format(self.service_mail_name))
write_in_file(path, conf)
logger.info("New {} configuration successfully applied"
"".format(self.service_mail_name))
# ### Write the first template (ssmtp.conf)
tpl, path = tpl_utils.get_template(self.service_name)
conf = tpl.render(conf=parameters, conf_service=service_settings)
logger.info("{} configuration has changed, writing new one".format(
self.service_name))
write_in_file(path, conf)
logger.info("New {} configuration successfully applied".format(
self.service_name))
def _send_template(self):
"""
Send template to the Elastcsearch Repository
Directive to not analyze string field
"""
t_access = tpl_utils.get_template('elasticsearch-access')[0]
t_pf = tpl_utils.get_template('elasticsearch-pf')[0]
t_vulture = tpl_utils.get_template('elasticsearch-vulture')[0]
t_diagnostic = tpl_utils.get_template('elasticsearch-diagnostic')[0]
templates_els = {
"vulture_access":
t_access.render(index_name=self.es_access_index_name),
"vulture_pf":
t_pf.render(index_name=self.es_packetfilter_index_name),
"vulture_logs":
t_vulture.render(index_name=self.es_vulturelogs_index_name),
"vulture_diagnostic":
t_diagnostic.render(index_name=self.es_diagnostic_index_name)
}
for host in self.es_host.split(','):
try:
for index_name, template in templates_els.items():
success = request(
'PUT',
"{}/_template/{}_template".format(host, index_name),
data=template,
headers={'Content-Type': 'application/json'})
if success.status_code != 200:
logger.info(
"Can't send {} template to ElasticSearch Node {}. Error: {}"
.format(index_name, host, success.text))
except Exception as e:
logger.info(
"Node Elasticsearch on host {} is down. Error: {}".format(
host, e))
continue
def build_configuration_files(listen_ip=None):
""" Check if Vulture GUI Apache configuration file has changed.
If changes are detected, a new one is writed.
:return:
"""
try:
# Looking for listen IP from DB data
if not listen_ip:
from gui.models.system_settings import Cluster
cluster = Cluster.objects.get()
node = cluster.get_current_node()
mngt_listener = node.get_management_listener()
listen_ip = mngt_listener.ip
parameters = {'listen_ip': listen_ip}
tpl, path = tpl_utils.get_template('gui')
conf = tpl.render(conf=parameters)
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
tpl, path = tpl_utils.get_template('portal')
conf = tpl.render(conf={})
try:
f = open(path, 'r')
orig_conf = f.read()
except IOError:
orig_conf = None
# Configuration file differs => writing new version
if orig_conf != conf:
write_in_file(path, conf)
return True
except Exception as e:
return False
def write_configuration(self, parameters, service_settings=None, text_config=None):
template, path_template = tpl_utils.get_template(self.service_name)
if not text_config:
try:
parameters = self.write_certificate(parameters)
except Exception as e:
logger.error("Failed to write Zabbix certificate files :")
logger.exception(e)
raise ServiceConfigError(str(e))
text_config = template.render({'conf': parameters})
logger.info("[ZABBIX] Writing configuration file '{}'".format(path_template))
try:
# The called function must raise !
self.write_configuration_file(path_template, text_config)
except Exception as e:
logger.error("Failed to write Zabbix configuration file '{}' :".format(path_template))
logger.exception(e)
raise ServiceConfigError(str(e))
def write_configuration(self,
parameters,
service_settings=None,
text_config=None):
""" Write service configuration into system configuration file.
Write operation occurs if configuration differs from initial file
:param parameters: A dict containing configuration parameters
:param service_settings: service settings used to generate template
:param text_config: Config to use instead of template
"""
tpl, path = tpl_utils.get_template(self.service_name)
if text_config:
try:
conf_sha1 = sha1(text_config.encode('utf8')).hexdigest()
except Exception as e:
return False
# Write conf to temporary file
with open('/tmp/{}.conf'.format(conf_sha1), 'w') as f:
f.write(text_config)
# Set owner of that file : root:wheel
try:
os_system(
"/usr/local/bin/sudo -u vlt-sys /usr/local/bin/sudo /usr/local/bin/sudo "
"/usr/sbin/chown root:wheel /tmp/{}.conf".format(
conf_sha1))
except Exception as e:
logger.error(
"Unable to set owner root:wheel of file '/tmp/{}.conf', error: {}"
.format(conf_sha1, e))
# Mv the temporary file to configuration file
proc = Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'/usr/local/bin/sudo', '/bin/mv',
'/tmp/{}.conf'.format(conf_sha1), path
],
stdout=PIPE)
res, errors = proc.communicate()
if not errors:
logger.info("New {} configuration successfully applied".format(
self.service_name))
return True
else:
logger.info("Problem during write of {} configuration".format(
self.service_name))
return False
elif self.conf_has_changed(parameters,
service_settings,
conf_django=settings):
conf = tpl.render(conf=parameters,
conf_service=service_settings,
conf_django=settings)
if conf:
logger.info(
"{} configuration has changed, writing new one".format(
self.service_name))
if write_in_file(path, conf):
logger.info(
"New {} configuration successfully applied".format(
self.service_name))
return True
else:
return False
def getConf(self):
parameters = dict()
ret = dict()
ret['error'] = ""
try:
parameters['nrclass'] = str(np.array(self.n_support_).shape[0])
parameters['SVdims'] = str(
np.array(self.support_vectors_).shape[0]) + " " + str(
np.array(self.support_vectors_).shape[1])
parameters['SV'] = little_endian(
str(binascii.hexlify(np.array(self.support_vectors_).data)),
16)
try:
parameters['supportdims'] = str(
np.array(self.support_).shape[0]) + " " + str(
np.array(self.support_).shape[1])
except:
parameters['supportdims'] = str(
np.array(self.support_).shape[0]) + " 1"
parameters['support'] = little_endian(
str(binascii.hexlify(np.array(self.support_).data)), 8)
parameters['SVcoef'] = little_endian(
str(binascii.hexlify(np.array(self.dual_coef_).data)), 16)
parameters['rho'] = little_endian(
str(binascii.hexlify(np.array(self._intercept_).data)), 16)
parameters['nSV'] = little_endian(
str(binascii.hexlify(np.array(self.n_support_).data)), 8)
parameters['SVcoefstrides'] = str(
np.array(self._dual_coef_).strides[0]) + " " + str(
np.array(self._dual_coef_).strides[1])
parameters['gamma'] = str("%.17f" % self.gamma)
except Exception as e:
ret['error'] = str(e)
prefix = ""
if self.algo_used == "Levenstein":
svm = SVM.objects(id=ObjectId(self.id)).no_dereference().only(
'dataset_used').first()
liste_uris = Dataset.objects(
id=svm.dataset_used.id).only('uri').first().uri
parameters['uris'] = "/".join(liste_uris)
prefix = "svm2"
elif self.algo_used == "Levenstein2":
svm = SVM.objects(id=ObjectId(self.id)).no_dereference().only(
'dataset_used').first()
liste_uris = Dataset.objects(
id=svm.dataset_used.id).only('uri').first().uri
parameters['uris'] = ";".join(liste_uris)
prefix = "svm3"
elif self.algo_used == "HTTPcode_bytes_received":
prefix = "svm4"
elif self.algo_used == "Ratio":
prefix = "svm5"
if not prefix:
ret['error'] = [
'<br/>SVM algorithm not recognized or not compatible'
]
else:
t = tpl_utils.get_template("vulture_" + prefix)[0]
ret['config'] = t.render(conf=parameters)
return ret
def check_status():
""" Check if krb5.conf need updates. Update file if needed
:return: True if krb5.conf was updated, False otherwise
"""
# No matter if keytab need update : Do it !
# First ; Clear file (don't delete to not change permissions)
try:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'/home/vlt-sys/scripts/flush_keytab.sh'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
res, errors = proc.communicate()
logger.debug("CONF::Kerberos: Flushing keytab stdout: {}".format(
str(res)))
if errors:
logger.error("CONF::Kerberos: Flushing keytab error: {}".format(
str(errors)))
except Exception as e:
logger.error("CONF::Kerberos: Flushing keytab error: {}".format(
str(e)))
# retrieve the Kerberos repositories to generate the configuration file and the keytab
krb5_repos = list()
repo_list = BaseAbstractRepository.get_auth_repositories()
for repo in repo_list:
if isinstance(repo.get_backend(), KerberosBackend):
# Don't write bullshit in system files
if test_keytab(repo.keytab)['status']:
# Retrieve keytabs from repos and import them into master keytab
result = import_keytab(logger, repo.keytab)
if not result:
logger.error(
"CONF::Kerberos: Fail to import keytab for repository : {}"
.format(str(repo.repo_name)))
else:
logger.info(
"CONF::Kerberos: Successfully imported keytab for repository : {}"
.format(str(repo.repo_name)))
else:
logger.error(
"CONF::Kerberos: Unable to decode the keytab : write_on_system not allowed."
)
# And other attributes needed to generate the configuration file
krb5_repo = dict()
krb5_repo['realm'] = repo.realm
if str(repo.domain_realm)[0] == '.':
krb5_repo['domain'] = str(repo.domain_realm)[1:]
else:
krb5_repo['domain'] = repo.domain_realm
krb5_repo['kdcs'] = str(repo.kdc).split(',')
krb5_repo['admin_server'] = repo.admin_server
krb5_repos.append(krb5_repo)
if len(krb5_repos) > 0:
# Get required fields to generate the template with Kerberos Repositories
parameters = {
'default_realm': krb5_repos[0]['realm'],
'repos': krb5_repos,
}
# Generate the configuration with template and repositories retrieved
tpl, path_rc = tpl_utils.get_template('kerberos')
conf_rc = tpl.render(conf=parameters)
try:
f = open(path_rc, 'r')
orig_conf = f.read()
except IOError:
orig_conf = ""
# If krb5.conf and newly generated conf differs => writing new version
if orig_conf != conf_rc:
write_result = write_in_file(path_rc, conf_rc)
return True
else:
return False
return False
def check_status():
""" Check if rc.conf.local need updates. Update file if needed
:return: True if rc.conf.local was updated, False otherwise
"""
listeners = []
devices = {}
haproxy = False
reg = re.compile("^ifconfig_([^ ]+)_alias\{\}.*")
cluster = Cluster.objects.get()
node = cluster.get_current_node()
# We add alias number (by device)
for listener in node.get_listeners():
if reg.match(listener.rc_conf):
dev = reg.match(listener.rc_conf).groups()[0]
if not devices.get(dev):
devices[dev] = 0
cpt = devices.get(dev)
rc_conf = listener.rc_conf.format(cpt)
devices[dev] += 1
else:
rc_conf = listener.rc_conf
listeners.append(rc_conf)
""" GUI-0.3 Upgrade
system_settings.pf_settings may not exists in GUI-0.3 upgraded from GUI-0.2
In this case we need to initialize it with default VALUES
"""
system_settings = node.system_settings
if system_settings.pf_settings is None:
system_settings.pf_settings = PFSettings()
if system_settings.ipsec_settings is None:
system_settings.ipsec_settings = IPSECSettings()
loadbalancers = []
for loadbalancer in Loadbalancer.objects.all():
if loadbalancer.ssl_profile and loadbalancer.ssl_profile.hpkp_enable:
loadbalancer.ssl_profile.pkp = PKP.getSPKIFingerpring(
loadbalancer.ssl_profile.certificate.cert)
if loadbalancer.incoming_listener.is_carp:
for l in loadbalancer.incoming_listener.get_related_carp_inets():
if l.get_related_node() == node:
haproxy = True
if loadbalancer not in loadbalancers:
loadbalancers.append(loadbalancer)
elif loadbalancer.incoming_listener.get_related_node() == node:
if loadbalancer not in loadbalancers:
loadbalancers.append(loadbalancer)
haproxy = True
try:
pf_logs = True if system_settings.pf_settings.repository_type == 'data' else False
except KeyError:
pf_logs = False
parameters = {
'default_ipv4_gw': node.default_ipv4_gw,
'default_ipv6_gw': node.default_ipv6_gw,
'static_route': node.static_route,
'listeners': listeners,
'pf_logs': pf_logs,
'haproxy': haproxy,
'strongswan': system_settings.ipsec_settings.enabled
}
# Checking if rc.conf.local differs.
tpl, path_rc = tpl_utils.get_template('rc.conf.local')
conf_rc = tpl.render(conf=parameters)
try:
with open(path_rc, 'r') as f:
orig_conf = f.read()
except IOError:
orig_conf = ""
# rc.conf.local and/or pf.conf differs => writing new version
if orig_conf != conf_rc:
write_in_file(path_rc, conf_rc)
if node.default_ipv4_gw or node.default_ipv6_gw:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'service', 'routing', 'restart'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
#### HAPROXY Configuration ####
parameters = {'haproxy': loadbalancers}
tpl, path_haproxy = tpl_utils.get_template('vlthaproxy')
conf_haproxy = tpl.render(conf=parameters)
if not len(loadbalancers):
conf_haproxy = ""
try:
with open(path_haproxy, 'r') as f:
orig_conf = f.read()
except IOError:
orig_conf = ""
## Check diff in haproxy.conf
proc = subprocess.Popen([
"/usr/local/bin/sudo", "-u", "vlt-sys", "/usr/local/bin/sudo",
"service", "vlthaproxy", "status"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
res, errors = proc.communicate()
haproxy_need_start = "haproxy not running" in errors.decode('utf8')
if haproxy and orig_conf != conf_haproxy:
write_in_file(path_haproxy, conf_haproxy)
""" Delete all certificates in /home/vlt-sys/Engine/conf/haproxy """
os.system("/bin/rm %shaproxy/*" % settings.CONF_DIR)
""" Write SSL-Profile certificates """
for loadbalancer in loadbalancers:
if loadbalancer.enable_tls and loadbalancer.ssl_profile:
loadbalancer.ssl_profile.write_HAProxy_conf()
""" Store certificates so that HA-PROXY may use them """
for cert in SSLCertificate.objects():
cert.write_certificate()
if not haproxy_need_start:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'service', 'vlthaproxy', 'restart'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
else:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'service', 'vlthaproxy', 'start'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
elif haproxy and haproxy_need_start:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'service', 'vlthaproxy', 'start'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
elif not haproxy:
write_in_file(path_haproxy, conf_haproxy)
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'service', 'vlthaproxy', 'stop'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc.communicate()
cluster = Cluster.objects.get()
node = cluster.get_current_node()
## CHECK Vulture Cluster Whitelist for Packet Filter
ips = []
for listener in Listener.objects.all():
ips.append(listener.ip)
ips = set(ips)
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo', 'pfctl',
'-t', 'vulture_cluster', '-T', 'show'
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
res, errors = proc.communicate()
if res is False:
return False
res = res.decode('utf8')
for ip in ips:
if ip not in node.system_settings.pf_settings.pf_whitelist:
node.system_settings.pf_settings.pf_whitelist = node.system_settings.pf_settings.pf_whitelist + '\n' + str(
ip)
node.save(bootstrap=True)
res = [r.strip() for r in res.split('\n') if r != ""]
for ip in ips:
if ip not in res:
proc = subprocess.Popen([
'/usr/local/bin/sudo', '-u', 'vlt-sys', '/usr/local/bin/sudo',
'pfctl', '-t', 'vulture_cluster', '-T', 'add', ip
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
res2, errors = proc.communicate()
if res2 is False:
return False
return True