def reconcile():
# MONGO #
m = MongoBase()
if not m.connect():
return False
m.connect_primary()
# REDIS #
r = RedisBase()
master_node = r.get_master()
r = RedisBase(node=master_node)
redis_list_name = "logs_darwin"
ignored_alerts = list()
rangeLen = r.redis.llen(redis_list_name)
alerts = r.redis.lrange(redis_list_name, "0", str(rangeLen - 1))
r.redis.ltrim(redis_list_name, str(rangeLen), "-1")
for alert in alerts:
alert = str(alert, "utf-8")
a = json.loads(alert)
evt_id = a.get("evt_id")
if evt_id is None:
ignored_alerts.append(a)
continue
query = {"darwin_id": evt_id}
newvalue = {
"$set": {
"darwin_alert_details": a,
"darwin_is_alert": True
}
}
m.update_one("logs", query, newvalue)
return True
def is_master_redis(self):
"""
Check if the current Node is master or not
:return: True / False, or None in case of a failure
"""
if self.management_ip:
c = RedisBase()
master_node = c.get_master(self.name)
if master_node == self.name:
return True
elif master_node:
return False
return None
def reconcile(self):
node = Cluster.get_current_node()
if not node.is_master_mongo:
return False
mongo = MongoBase()
if not mongo.connect():
return False
mongo.connect_primary()
redis = RedisBase()
master_node = redis.get_master()
redis = RedisBase(node=master_node)
filepath = ALERTS_FILE
# Pops alerts produced when vulture was down
# Do not retry, as there is likely no cache for remaining alerts in current Redis
self.pops(mongo, redis, filepath, max_tries=1)
if self.shutdown_flag.is_set():
return True
redis_channel = REDIS_CHANNEL
listener = redis.redis.pubsub()
listener.subscribe([redis_channel])
logger.info(
"Reconcile: start listening {} channel.".format(redis_channel))
while not self.shutdown_flag.is_set():
alert = listener.get_message(ignore_subscribe_messages=True,
timeout=2)
# If we have no messages, alert is None
if alert:
# Only use the channel to trigger popping alerts
self.pops(mongo, redis, filepath)
return True
def cluster_join(master_hostname, master_ip, secret_key, ca_cert=None, cert=None, key=None):
"""
Join an existing cluster
:param master_hostname: Master hostname
:param master_ip: Master management Ip address
:param secret_key: The node's secret key
:param ca_cert: The CA Certificate (optional: can be retrieved automagically)
:param cert: The node certificate (optional: can be retrieved automagically)
:param key: The node private key (optional: can be retrieved automagically)
:return: True / False
"""
""" We are coming from the CLI interface """
try:
infos = requests.get(
"https://{}:8000/api/v1/system/cluster/info".format(master_ip),
headers={'Cluster-api-key': secret_key},
verify=False
).json()
if not infos['status']:
raise Exception('Error at API Request Cluster Info: {}'.format(infos['data']))
time_master = infos['data'][master_hostname]['unix_timestamp']
time_now = time.time()
if abs(time_now - time_master) > 60 or abs(time_now + time_master) < 60:
logger.info('Nodes not at the same date. Please sync with NTP Server')
print('Nodes not at the same date. Please sync with NTP Server')
return False
except Exception as e:
logger.error("Error at API Request Cluster Info: {} Invalid API KEY ?".format(e))
return False
if not ca_cert:
try:
infos = requests.post(
"https://{}:8000/api/system/pki/get_ca".format(master_ip),
headers={'Cluster-api-key': secret_key},
verify=False
).json()
ca_cert = infos.get('ca_cert')
except Exception as e:
logger.error("Unable to retrieve CA certificate: {}".format(e))
return False
""" We are coming from the CLI interface """
if not cert or not key:
try:
infos = requests.post(
"https://{}:8000/api/system/pki/get_cert/".format(master_ip),
headers={'Cluster-api-key': secret_key},
data={'node_name': get_hostname()},
verify=False
).json()
cert = infos.get('cert')
key = infos.get('key')
except Exception as e:
logger.error("Unable to retrieve Node certificate: {}".format(e))
return False
if cert and key:
bundle = cert + key
else:
logger.error("Unable to retrieve Node certificate and key, check secret key")
return False
with open("/var/tmp/ca.pem", "w") as f:
f.write(ca_cert)
with open("/var/tmp/node.cert", "w") as f:
f.write(cert)
with open("/var/tmp/node.key", "w") as f:
f.write(key)
with open("/var/tmp/node.pem", "w") as f:
f.write(bundle)
""" At this point we should have valid certificates:
Save them on system """
subprocess.check_output([
'/home/vlt-os/scripts/write_cert.sh'
])
""" At this point, certificates have been overwritten
=> We need to destroy replicaset & restart mongoDB
"""
logger.info("replDestroy: Restart Mongodb with new certificates")
mongo = MongoBase()
mongo.repl_destroy()
""" Ask primary to join us on our management IP """
# TODO: verify to true
infos = requests.post(
"https://{}:8000/api/system/cluster/add/".format(master_ip),
headers={'Cluster-api-key': secret_key},
data={'slave_ip': get_management_ip(), 'slave_name': get_hostname()},
verify=False
)
if infos.status_code != 200:
raise Exception("Error at API Call on /system/cluster/add/ Response code: {}".format(infos.status_code))
infos = infos.json()
if not infos.get('status'):
logger.error("Error during API Call to add node to cluster: {}".format(infos.get('message')))
return False
""" Join our redis server to the redis master """
c = RedisBase()
redis_master_node = c.get_master(master_ip)
c.slave_of(redis_master_node, 6379)
""" Tell local sentinel to monitor local redis server """
c = RedisBase(get_management_ip(), 26379)
c.sentinel_monitor()
""" Sleep a few seconds in order for the replication to occur """
time.sleep(3)
""" Create the local node """
try:
node = Node.objects.get(
name=get_hostname(),
management_ip=get_management_ip()
)
except Exception:
logger.error("cluster_join:: Unable to find slave node !")
logger.error("cluster_join:: Unable to find slave node !")
return False
""" Update uri of internal Log Forwarder """
logfwd = LogOMMongoDB.objects.get()
logfwd.uristr = mongo.get_replicaset_uri()
logfwd.save()
""" Save certificates on new node """
for cert in X509Certificate.objects.exclude(is_vulture_ca=True):
cert.save_conf()
""" Read network config and store it into mongo """
""" No rights to do that in jail - API request """
node.api_request('toolkit.network.network.refresh_nic')
""" Read ZFS file systems """
node.api_request('system.zfs.zfs.refresh')
""" Download reputation databases before crontab """
node.api_request("gui.crontab.feed.security_update")
""" And configure + restart netdata """
logger.debug("API call to netdata configure_node")
# API call to Cluster - to refresh nodes on each node conf
Cluster.api_request('services.netdata.netdata.configure_node')
logger.debug("API call to restart netdata service")
node.api_request('services.netdata.netdata.restart_service')
logger.debug("API call to configure HAProxy")
node.api_request("services.haproxy.haproxy.configure_node")
logger.debug("API call to write Darwin policies conf")
for policy in DarwinPolicy.objects.all():
node.api_request("services.darwin.darwin.write_policy_conf", policy.pk)
logger.debug("API call to configure Darwin")
node.api_request("services.darwin.darwin.build_conf")
# API call to while Cluster - to refresh Nodes list in conf
logger.debug("API call to update configuration of Apache GUI")
Cluster.api_request("services.apache.apache.reload_conf")
""" The method configure restart rsyslog if needed """
logger.debug("API call to configure rsyslog")
# API call to whole Cluster - to refresh mongodb uri in pf logs
Cluster.api_request("services.rsyslogd.rsyslog.configure_node")
logger.debug("API call to configure logrotate")
node.api_request("services.logrotate.logrotate.reload_conf")
return True