class RedisClient:
"""
A class to write BBB information on a REDIS server
"""
def __init__(
self,
path: str = CONFIG_PATH,
log_path: str = LOG_PATH_BBB,
):
# Configuring logging
self.logger = logging.getLogger("bbbread")
self.logger.setLevel(logging.INFO)
formatter = logging.Formatter("%(levelname)s:%(asctime)s:%(name)s:%(message)s")
file_handler = RotatingFileHandler(log_path, maxBytes=15000000, backupCount=5)
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.logger.info("Starting BBBread up")
# Defining local and remote database
self.local_db = redis.StrictRedis(host="127.0.0.1", port=6379, socket_timeout=4)
self.logger.info("Searching for active database")
self.remote_db = self.find_active()
# Defining BBB object and formatting remote hash name as "BBB:IP_ADDRESS:HOSTNAME"
if CONFIG_PATH != path or LOG_PATH_BBB != log_path:
self.bbb = BBB(path=path, logfile=log_path)
else:
self.bbb = node
self.nw_service = None
for service in subprocess.check_output(["connmanctl", "services"]).decode().split("\n")[:-1]:
if "Wired" in service:
self.nw_service = service.split(16 * " ")[1]
break
self.l_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.bbb_ip_type, self.bbb_ip, self.bbb_nameservers = self.get_network_specs()
self.bbb_hostname = socket.gethostname()
self.local_db.hmset(
"device",
{
"name": self.bbb_hostname,
"sector": self.bbb.node.sector,
"details": self.bbb.node.details,
"state_string": self.bbb.node.state_string,
"state": self.bbb.node.state,
"ip_type": self.bbb_ip_type,
"ip_address": self.bbb_ip,
"nameservers": self.bbb_nameservers,
},
)
self.hashname = f"BBB:{self.bbb_ip}:{self.bbb_hostname}"
self.command_listname = f"{self.hashname}:Command"
self.remote_db.hmset(self.hashname, self.local_db.hgetall("device"))
# Pinging thread
self.ping_thread = threading.Thread(target=self.ping_remote, daemon=True)
self.ping_thread.start()
self.logger.info("Pinging thread started")
# Listening thread
self.listening = True
self.logger.info("Listening thread started")
self.logger.info("BBBread startup completed")
self.logs_name = f"{self.hashname}:Logs"
self.listen()
def get_network_specs(self):
nameservers = "0.0.0.0"
ip_type = "0.0.0.0"
ip_address = "0.0.0.0"
if not self.nw_service:
try:
self.l_socket.connect(("10.255.255.255", 1))
except OSError:
self.l_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
return ip_type, ip_address, nameservers
ip_address, ip_type = self.l_socket.getsockname()[0], "dhcp"
return ip_type, ip_address, nameservers
command_out = subprocess.check_output(["connmanctl", "services", self.nw_service]).decode().split("\n")[:-1]
if command_out:
for line in command_out:
# Address line
if "IPv4 = " in line:
try:
ip_type = line[18 : line.index(",")]
ip_address = line[line.index("Address=") + 8 : line.index("Netmask") - 2]
except Exception:
continue
# Nameservers line
if "Nameservers = " in line:
try:
nameservers = line[line.index("=") + 4 : -2]
except Exception:
continue
return ip_type, ip_address, nameservers
def find_active(self):
"""Find available server and replace old one"""
while True:
for server in SERVER_LIST:
try:
remote_db = redis.StrictRedis(host=server, port=6379, socket_timeout=4)
remote_db.ping()
self.logger.info(f"Connected to {server} Redis Server")
return remote_db
except redis.exceptions.ConnectionError:
self.logger.warning(f"{server} Redis server is disconnected")
except Exception as e:
self.logger.warning(f"Could not connect to {server}: {e}")
time.sleep(50)
continue
self.logger.info("Server not found. Retrying to connect in 10 seconds...")
time.sleep(10)
def ping_remote(self):
"""Thread that updates remote database every 10s, if pinging is enabled"""
cycles_since_heavy_operation = 0
disk_usage = shutil.disk_usage("/")
percent_disk_usage = disk_usage.used / disk_usage.total
while True:
try:
new_ip_type, new_ip, new_nameservers = self.get_network_specs()
new_hostname = socket.gethostname()
if not self.remote_db.hexists(self.hashname, "ip_address"):
self.remote_db.hmset(self.hashname, self.local_db.hgetall("device"))
if cycles_since_heavy_operation > 16:
disk_usage = shutil.disk_usage("/")
percent_disk_usage = disk_usage.used / disk_usage.total
if percent_disk_usage > 90:
self.log_remote("Disk usage is at {}%".format(percent_disk_usage), self.logger.warning)
cycles_since_heavy_operation = 0
else:
cycles_since_heavy_operation += 1
self.remote_db.hmset(
self.hashname, {"heartbeat": 1, "disk_usage": "{:.2f}%".format(percent_disk_usage * 100)}
)
# Formats remote hash name as "BBB:IP_ADDRESS"
if (
new_ip != self.bbb_ip
or new_hostname != self.bbb_hostname
or new_ip_type != self.bbb_ip_type
or new_nameservers != self.bbb_nameservers
):
info = self.local_db.hgetall("device")
self.hashname = f"BBB:{new_ip}:{new_hostname}"
old_hashname = f"BBB:{self.bbb_ip}:{self.bbb_hostname}"
old_info = info.copy()
old_info.update(
{
b"state_string": self.hashname,
b"name": self.bbb_hostname,
b"ip_address": self.bbb_ip,
b"ip_type": self.bbb_ip_type,
b"nameservers": self.bbb_nameservers,
}
)
try:
self.remote_db.rename(f"{old_hashname}:Command", f"{self.hashname}:Command")
self.remote_db.rename(f"{old_hashname}:Logs", f"{self.hashname}:Logs")
except redis.exceptions.ResponseError:
pass
self.logger.info(old_info)
self.remote_db.hmset(old_hashname, old_info)
self.listening = True
self.bbb_ip, self.bbb_hostname, self.bbb_ip_type, self.bbb_nameservers = (
new_ip,
new_hostname,
new_ip_type,
new_nameservers,
)
self.logs_name = f"{self.hashname}:Logs"
self.command_listname = f"{self.hashname}:Command"
info.update(
{
b"name": new_hostname,
b"ip_address": new_ip,
b"ip_type": new_ip_type,
b"nameservers": new_nameservers,
}
)
self.remote_db.hmset(self.hashname, info)
time.sleep(10)
except Exception as e:
self.logger.error(f"Pinging thread found an exception: {e}")
time.sleep(10)
self.find_active()
def listen(self):
"""Thread to process server's commands"""
while True:
time.sleep(3)
if not self.listening:
time.sleep(2)
continue
if not self.ping_thread.is_alive():
break
try:
if self.remote_db.exists(self.command_listname):
command = self.remote_db.lpop(self.command_listname).decode()
command = command.split(";")
command[0] = int(command[0])
else:
continue
except redis.exceptions.TimeoutError:
self.logger.error("Reconnecting to Redis server")
time.sleep(1)
continue
except ValueError:
self.logger.error("Failed to convert first part of the command to integer")
continue
except Exception as e:
self.logger.error(f"Listening thread found an exception: {e}")
time.sleep(3)
continue
self.logger.info(f"Command received {command}")
if command[0] == Command.REBOOT:
self.log_remote("Reboot command received", self.logger.info)
self.bbb.reboot()
elif command[0] == Command.SET_HOSTNAME and len(command) == 2:
new_hostname = command[1]
self.bbb.update_hostname(new_hostname)
# Updates variable names
self.log_remote(f"Hostname changed to {new_hostname}", self.logger.info)
self.listening = False
elif command[0] == Command.SET_IP:
ip_type = command[1]
# Verifies if IP is to be set manually
if ip_type == "manual" and len(command) == 5:
new_ip, new_mask, new_gateway = command[2:]
self.bbb.update_ip_address(ip_type, new_ip, new_mask, new_gateway)
# Updates variable names
info = f"IP manually changed to {new_ip}, netmask {new_mask}, gateway {new_gateway}"
self.log_remote(info, self.logger.info)
self.listening = False
# Verifies if IP is DHCP
elif ip_type == "dhcp":
self.bbb.update_ip_address(ip_type)
# Updates variable names
time.sleep(1)
self.log_remote("IP changed to DHCP", self.logger.info)
self.listening = False
elif command[0] == Command.SET_NAMESERVERS and len(command) == 3:
nameserver_1, nameserver_2 = command[1:]
self.log_remote(f"Nameservers changed: {nameserver_1}, {nameserver_2}", self.logger.info)
self.bbb.update_nameservers(nameserver_1, nameserver_2)
elif command[0] >= Command.RESTART_SERVICE and len(command) == 2:
action = "stop" if command[0] == Command.STOP_SERVICE else "restart"
service_name = command[1]
self.log_remote(f"{service_name} service {action}", self.logger.info)
subprocess.check_output(["systemctl", action, service_name])
def log_remote(self, message: str, log_level: Callable):
"""Pushes logs to remote server"""
try:
log_level(message)
self.remote_db.hset(self.logs_name, int(time.time()), message)
except Exception as e:
self.logger.error(f"Failed to send remote log information: {e}")
class RedisClient:
"""
A class to write BBB information on a REDIS server
"""
def __init__(self,
path=CONFIG_PATH,
remote_host_1=SERVER_IP,
remote_host_2=BACKUP_SERVER,
log_path=LOG_PATH_BBB):
# Configuring logging
self.logger = logging.getLogger('bbbread')
self.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(levelname)s:%(asctime)s:%(name)s:%(message)s')
file_handler = logging.FileHandler(log_path)
file_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.logger.debug("Starting BBBread up")
# Defining local and remote database
self.local_db = redis.StrictRedis(host='127.0.0.1',
port=6379,
socket_timeout=2)
self.remote_db_1 = redis.StrictRedis(host=remote_host_1,
port=6379,
socket_timeout=2)
self.remote_db_2 = redis.StrictRedis(host=remote_host_2,
port=6379,
socket_timeout=2)
self.logger.debug("Searching for active database")
self.remote_db = self.find_active()
# Defining BBB object and formatting remote hash name as "BBB:IP_ADDRESS:HOSTNAME"
self.bbb = BBB(path)
update_local_db()
self.bbb_ip, self.bbb_hostname = self.local_db.hmget(
'device', 'ip_address', 'name')
self.bbb_ip = self.bbb_ip.decode()
self.bbb_hostname = self.bbb_hostname.decode()
self.hashname = "BBB:{}:{}".format(self.bbb_ip, self.bbb_hostname)
self.command_listname = "BBB:{}:{}:Command".format(
self.bbb_ip, self.bbb_hostname)
# Pinging thread
self.ping_thread = threading.Thread(target=self.ping_remote,
daemon=True)
self.pinging = True
self.ping_thread.start()
self.logger.debug("Pinging thread started")
# Listening thread
self.listen_thread = threading.Thread(target=self.listen, daemon=True)
self.listening = True
self.listen_thread.start()
self.logger.debug("Listening thread started")
self.logger.debug("BBBread startup completed")
def find_active(self):
try:
self.remote_db_1.ping()
self.logger.debug("Connected to LA-RaCtrl-CO-Srv-1 Redis Server")
return self.remote_db_1
except redis.exceptions.ConnectionError:
try:
self.remote_db_2.ping()
self.logger.debug(
"Connected to CA-RaCtrl-CO-Srv-1 Redis Server")
return self.remote_db_2
except redis.exceptions.ConnectionError:
self.logger.critical("No remote database found")
raise Exception("No remote database found")
def ping_remote(self):
"""Thread that updates remote database every 10s, if pinging is enabled"""
while True:
if not self.pinging:
time.sleep(2)
continue
try:
self.force_update()
time.sleep(10)
except Exception as e:
self.logger.error("Pinging Thread died:\n{}".format(e))
time.sleep(1)
self.find_active()
def listen(self):
"""Thread to process server's commands"""
while True:
time.sleep(2)
if not self.listening:
time.sleep(2)
continue
try:
self.command_listname = self.hashname + ":Command"
if self.remote_db.keys(self.command_listname):
command = self.remote_db.lpop(
self.command_listname).decode()
command = command.split(";")
# Verifies if command is an integer
try:
command[0] = int(command[0])
except ValueError:
self.logger.error(
"Failed to convert first part of the command to integer"
)
continue
self.logger.info("command received {}".format(command))
if command[0] == Command.REBOOT:
self.logger.info("Reboot command received")
self.bbb.reboot()
elif command[0] == Command.SET_HOSTNAME and len(
command) == 2:
new_hostname = command[1]
self.bbb.update_hostname(new_hostname)
# Updates variable names
self.logger.info("Hostname changed to " + new_hostname)
self.listening = False
elif command[0] == Command.SET_IP:
ip_type = command[1]
# Verifies if IP is to be set manually
if ip_type == 'manual' and len(command) == 5:
new_ip = command[2]
new_mask = command[3]
new_gateway = command[4]
self.bbb.update_ip_address(ip_type, new_ip,
new_mask, new_gateway)
# Updates variable names
self.logger.info("IP manually changed to {},"
"netmask {}, gateway {}".format(
new_ip, new_mask,
new_gateway))
self.listening = False
# Verifies if IP is DHCP
elif ip_type == 'dhcp':
self.bbb.update_ip_address(ip_type)
# Updates variable names
time.sleep(1)
self.logger.info("IP changed to DHCP")
self.listening = False
elif command[0] == Command.SET_NAMESERVERS and len(
command) == 3:
nameserver_1 = command[1]
nameserver_2 = command[2]
self.bbb.update_nameservers(nameserver_1, nameserver_2)
self.logger.info("Nameservers changed: {}, {}".format(
nameserver_1, nameserver_2))
elif command[0] == Command.STOP_SERVICE and len(
command) == 2:
service_name = command[1]
if service_name == 'bbbread':
self.logger.warning("Stopping BBBread")
subprocess.check_output(
['systemctl', 'stop', service_name])
self.logger.info(
"{} service stopped".format(service_name))
elif command[0] == Command.RESTART_SERVICE and len(
command) == 2:
service_name = command[1]
if service_name == 'bbbread':
self.logger.warning("Restarting BBBread")
subprocess.check_output(
['systemctl', 'restart', service_name])
self.logger.info(
"{} service restarted".format(service_name))
except Exception as e:
self.logger.error("Listening Thread died:\n{}".format(e))
time.sleep(1)
self.find_active()
continue
def force_update(self, log=False):
"""Updates local and remote database"""
if log:
self.logger.info("updating local db")
new_ip, new_hostname = update_local_db()
if log:
self.logger.info("local db updated")
info = self.local_db.hgetall('device')
# Formats remote hash name as "BBB:IP_ADDRESS"
self.hashname = "BBB:{}:{}".format(new_ip, new_hostname)
if new_ip != self.bbb_ip or new_hostname != self.bbb_hostname:
old_hashname = 'BBB:{}:{}'.format(self.bbb_ip, self.bbb_hostname)
old_info = info.copy()
old_info[b'state_string'] = self.hashname
old_info[b'name'] = self.bbb_hostname
old_info[b'ip_address'] = self.bbb_ip
if self.remote_db.keys(old_hashname + ":Command"):
self.remote_db.rename(old_hashname + ":Command",
self.hashname + ":Command")
self.logger.info(
"old ip: {}, new ip: {}, old hostname: {}, new hostname: {}".
format(self.bbb_ip, new_ip, self.bbb_hostname, new_hostname))
self.remote_db.hmset(old_hashname, old_info)
self.listening = True
# Updates remote hash
if log:
self.logger.info("updating remote db")
self.remote_db.hmset(self.hashname, info)
self.bbb_ip, self.bbb_hostname = (new_ip, new_hostname)