def reconcile(): # MONGO # m = MongoBase() if not m.connect(): return False m.connect_primary() # REDIS # r = RedisBase() master_node = r.get_master() r = RedisBase(node=master_node) redis_list_name = "logs_darwin" ignored_alerts = list() rangeLen = r.redis.llen(redis_list_name) alerts = r.redis.lrange(redis_list_name, "0", str(rangeLen - 1)) r.redis.ltrim(redis_list_name, str(rangeLen), "-1") for alert in alerts: alert = str(alert, "utf-8") a = json.loads(alert) evt_id = a.get("evt_id") if evt_id is None: ignored_alerts.append(a) continue query = {"darwin_id": evt_id} newvalue = { "$set": { "darwin_alert_details": a, "darwin_is_alert": True } } m.update_one("logs", query, newvalue) return True
def is_master_redis(self): """ Check if the current Node is master or not :return: True / False, or None in case of a failure """ if self.management_ip: c = RedisBase() master_node = c.get_master(self.name) if master_node == self.name: return True elif master_node: return False return None
def reconcile(self): node = Cluster.get_current_node() if not node.is_master_mongo: return False mongo = MongoBase() if not mongo.connect(): return False mongo.connect_primary() redis = RedisBase() master_node = redis.get_master() redis = RedisBase(node=master_node) filepath = ALERTS_FILE # Pops alerts produced when vulture was down # Do not retry, as there is likely no cache for remaining alerts in current Redis self.pops(mongo, redis, filepath, max_tries=1) if self.shutdown_flag.is_set(): return True redis_channel = REDIS_CHANNEL listener = redis.redis.pubsub() listener.subscribe([redis_channel]) logger.info( "Reconcile: start listening {} channel.".format(redis_channel)) while not self.shutdown_flag.is_set(): alert = listener.get_message(ignore_subscribe_messages=True, timeout=2) # If we have no messages, alert is None if alert: # Only use the channel to trigger popping alerts self.pops(mongo, redis, filepath) return True
def post(self, request, object_id, **kwargs): confirm = request.POST.get('confirm') if confirm == 'yes': try: obj_inst = self.obj.objects.get(pk=object_id) except ObjectDoesNotExist: return HttpResponseForbidden("Injection detected") """ Before Deleting the node we need to remove it from mongoDB """ c = MongoBase() c.connect() c.connect_primary() c.repl_remove(obj_inst.name + ":9091") """ Before Deleting the node we need to remove it from Redis """ c = RedisBase(obj_inst.management_ip) c.slave_of('NO', 'ONE') # Fixme: Cleanup Sentinel ? """ Let's rock """ obj_inst.delete() return HttpResponseRedirect(self.redirect_url)
def cluster_create(admin_user=None, admin_password=None): """ Initialize a new cluster from scratch This requires a valid hostname and existing certificates :param logger: A valid logger :return: True / False """ """ We're comming from CLI: Abort if cluster already exists """ try: Cluster.objects.get() logger.error("Error: Cluster already exists") return False except Exception: pass """ Create Cluster object in database """ cluster = Cluster() """ Create the local node """ node, new_node = Node.objects.get_or_create( name=get_hostname(), management_ip=get_management_ip() ) if new_node: logger.info("Registering new node '{}'".format(node.name)) node.internet_ip = node.management_ip node.save() """ Read network config and store it into mongo """ """ No rights to do that in jail - API request """ node.api_request('toolkit.network.network.refresh_nic') """ Read ZFS file systems """ node.api_request('system.zfs.zfs.refresh') """ Obtain Certificates and store it into mongoDB """ with open("/var/db/pki/ca.pem") as f: ca_cert = f.read() with open("/var/db/pki/ca.key") as f: ca_key = f.read() with open("/var/db/pki/node.cert") as f: cert = f.read() with open("/var/db/pki/node.key") as f: key = f.read() internal_ca = X509Certificate( is_vulture_ca=True, is_ca=True, is_external=False, status='V', cert=ca_cert, key=ca_key, serial=1 ) ca_name = internal_ca.explose_dn()['CN'] internal_ca.name = ca_name internal_ca.save() node_cert = X509Certificate( is_vulture_ca=False, is_ca=False, is_external=False, status='V', cert=cert, key=key, chain=ca_cert, serial=2 ) node_name = node_cert.explose_dn()['CN'] node_cert.name = node_name node_cert.save() """ Save cluster here, in case of error """ cluster.save() """ Generate random API Key for cluster management and NMAP """ """ Config object can not exists yet """ system_config = cluster.get_global_config() system_config.cluster_api_key = get_random_string( 16, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.,+') system_config.portal_cookie_name = get_random_string( 8, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') system_config.public_token = get_random_string(16, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') system_config.set_logs_ttl() system_config.save() for name in ('Administrator', 'Log Viewer'): Group.objects.get_or_create( name=name ) """ Delete any existing user """ User.objects.all().delete() if admin_user and admin_password: user = User.objects.create_superuser(admin_user, 'changeme@localhost', admin_password) user.save() time.sleep(5) """ Tell local sentinel to monitor local redis server """ c = RedisBase(get_management_ip(), 26379) try: # It may failed if Redis / Sentinel is already configured c.sentinel_monitor() except Exception as e: logger.error("Install::Sentinel monitor: Error: ") logger.exception(e) """ Update uri of internal Log Forwarder """ logfwd = LogOMMongoDB.objects.get() logfwd.uristr = "mongodb://{}:9091/?replicaset=Vulture&ssl=true".format(get_hostname()) logfwd.x509_certificate = node_cert logfwd.save() """ Create default TLSProfile """ tls_profile = TLSProfile(name="Default TLS profile", x509_certificate=node_cert) # All vars by default tls_profile.save() """ Download reputation databases before crontab """ node.api_request("gui.crontab.feed.security_update") """ And configure + restart netdata """ logger.debug("API call to netdata configure_node") node.api_request('services.netdata.netdata.configure_node') logger.debug("API call to restart netdata service") node.api_request('services.netdata.netdata.restart_service') """ No need to restart rsyslog because the conf is not complete """ logger.debug("API call to configure rsyslog") node.api_request("services.rsyslogd.rsyslog.build_conf") node.api_request("services.rsyslogd.rsyslog.configure_node") node.api_request("services.rsyslogd.rsyslog.restart_service") logger.debug("API call to configure HAProxy") node.api_request("services.haproxy.haproxy.configure_node") logger.debug("API call to write default Darwin filters conf") for policy in DarwinPolicy.objects.all(): node.api_request("services.darwin.darwin.write_policy_conf", policy.pk) logger.debug("API call to configure Darwin default policy") node.api_request("services.darwin.darwin.build_conf") logger.debug("API call to configure Apache GUI") node.api_request("services.apache.apache.reload_conf") logger.debug("API call to configure Logrotate") node.api_request("services.logrotate.logrotate.reload_conf")
def cluster_join(master_hostname, master_ip, secret_key, ca_cert=None, cert=None, key=None): """ Join an existing cluster :param master_hostname: Master hostname :param master_ip: Master management Ip address :param secret_key: The node's secret key :param ca_cert: The CA Certificate (optional: can be retrieved automagically) :param cert: The node certificate (optional: can be retrieved automagically) :param key: The node private key (optional: can be retrieved automagically) :return: True / False """ """ We are coming from the CLI interface """ try: infos = requests.get( "https://{}:8000/api/v1/system/cluster/info".format(master_ip), headers={'Cluster-api-key': secret_key}, verify=False ).json() if not infos['status']: raise Exception('Error at API Request Cluster Info: {}'.format(infos['data'])) time_master = infos['data'][master_hostname]['unix_timestamp'] time_now = time.time() if abs(time_now - time_master) > 60 or abs(time_now + time_master) < 60: logger.info('Nodes not at the same date. Please sync with NTP Server') print('Nodes not at the same date. Please sync with NTP Server') return False except Exception as e: logger.error("Error at API Request Cluster Info: {} Invalid API KEY ?".format(e)) return False if not ca_cert: try: infos = requests.post( "https://{}:8000/api/system/pki/get_ca".format(master_ip), headers={'Cluster-api-key': secret_key}, verify=False ).json() ca_cert = infos.get('ca_cert') except Exception as e: logger.error("Unable to retrieve CA certificate: {}".format(e)) return False """ We are coming from the CLI interface """ if not cert or not key: try: infos = requests.post( "https://{}:8000/api/system/pki/get_cert/".format(master_ip), headers={'Cluster-api-key': secret_key}, data={'node_name': get_hostname()}, verify=False ).json() cert = infos.get('cert') key = infos.get('key') except Exception as e: logger.error("Unable to retrieve Node certificate: {}".format(e)) return False if cert and key: bundle = cert + key else: logger.error("Unable to retrieve Node certificate and key, check secret key") return False with open("/var/tmp/ca.pem", "w") as f: f.write(ca_cert) with open("/var/tmp/node.cert", "w") as f: f.write(cert) with open("/var/tmp/node.key", "w") as f: f.write(key) with open("/var/tmp/node.pem", "w") as f: f.write(bundle) """ At this point we should have valid certificates: Save them on system """ subprocess.check_output([ '/home/vlt-os/scripts/write_cert.sh' ]) """ At this point, certificates have been overwritten => We need to destroy replicaset & restart mongoDB """ logger.info("replDestroy: Restart Mongodb with new certificates") mongo = MongoBase() mongo.repl_destroy() """ Ask primary to join us on our management IP """ # TODO: verify to true infos = requests.post( "https://{}:8000/api/system/cluster/add/".format(master_ip), headers={'Cluster-api-key': secret_key}, data={'slave_ip': get_management_ip(), 'slave_name': get_hostname()}, verify=False ) if infos.status_code != 200: raise Exception("Error at API Call on /system/cluster/add/ Response code: {}".format(infos.status_code)) infos = infos.json() if not infos.get('status'): logger.error("Error during API Call to add node to cluster: {}".format(infos.get('message'))) return False """ Join our redis server to the redis master """ c = RedisBase() redis_master_node = c.get_master(master_ip) c.slave_of(redis_master_node, 6379) """ Tell local sentinel to monitor local redis server """ c = RedisBase(get_management_ip(), 26379) c.sentinel_monitor() """ Sleep a few seconds in order for the replication to occur """ time.sleep(3) """ Create the local node """ try: node = Node.objects.get( name=get_hostname(), management_ip=get_management_ip() ) except Exception: logger.error("cluster_join:: Unable to find slave node !") logger.error("cluster_join:: Unable to find slave node !") return False """ Update uri of internal Log Forwarder """ logfwd = LogOMMongoDB.objects.get() logfwd.uristr = mongo.get_replicaset_uri() logfwd.save() """ Save certificates on new node """ for cert in X509Certificate.objects.exclude(is_vulture_ca=True): cert.save_conf() """ Read network config and store it into mongo """ """ No rights to do that in jail - API request """ node.api_request('toolkit.network.network.refresh_nic') """ Read ZFS file systems """ node.api_request('system.zfs.zfs.refresh') """ Download reputation databases before crontab """ node.api_request("gui.crontab.feed.security_update") """ And configure + restart netdata """ logger.debug("API call to netdata configure_node") # API call to Cluster - to refresh nodes on each node conf Cluster.api_request('services.netdata.netdata.configure_node') logger.debug("API call to restart netdata service") node.api_request('services.netdata.netdata.restart_service') logger.debug("API call to configure HAProxy") node.api_request("services.haproxy.haproxy.configure_node") logger.debug("API call to write Darwin policies conf") for policy in DarwinPolicy.objects.all(): node.api_request("services.darwin.darwin.write_policy_conf", policy.pk) logger.debug("API call to configure Darwin") node.api_request("services.darwin.darwin.build_conf") # API call to while Cluster - to refresh Nodes list in conf logger.debug("API call to update configuration of Apache GUI") Cluster.api_request("services.apache.apache.reload_conf") """ The method configure restart rsyslog if needed """ logger.debug("API call to configure rsyslog") # API call to whole Cluster - to refresh mongodb uri in pf logs Cluster.api_request("services.rsyslogd.rsyslog.configure_node") logger.debug("API call to configure logrotate") node.api_request("services.logrotate.logrotate.reload_conf") return True