def database_periodic(self):
try:
(linux_dist, x, y) = platform.linux_distribution()
if (linux_dist == 'Ubuntu'):
popen_cmd = "grep -A 1 'data_file_directories:'" + \
" /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2"
else:
popen_cmd = "grep -A 1 'data_file_directories:'" + \
" /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2"
(cassandra_data_dir, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
cassandra_data_dir = cassandra_data_dir.strip()
analytics_dir = cassandra_data_dir + '/ContrailAnalytics'
if os.path.exists(analytics_dir):
popen_cmd = "set `df -Pk " + analytics_dir + " | grep %` && echo $3 |" + \
" cut -d'%' -f1"
self.stderr.write("popen_cmd is " + popen_cmd + "\n")
(disk_space_used, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
popen_cmd = "set `df -Pk " + analytics_dir + " | grep %` && echo $4 |" + \
" cut -d'%' -f1"
self.stderr.write("popen_cmd is " + popen_cmd + "\n")
(disk_space_available, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
popen_cmd = "set `du -skL " + analytics_dir + "` && echo $1 | cut -d'%' -f1"
self.stderr.write("popen_cmd is " + popen_cmd + "\n")
(analytics_db_size, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
self.fail_status_bits &= ~self.FAIL_STATUS_DISK_SPACE_NA
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(disk_space_used)
db_stat.disk_space_available_1k = int(disk_space_available)
db_stat.analytics_db_size_1k = int(analytics_db_size)
db_info.name = socket.gethostname()
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
else:
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
except:
sys.stderr.write("Failed to get database usage" + "\n")
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
cassandra_cli_cmd = "cassandra-cli --host " + self.hostip + \
" --batch < /dev/null | grep 'Connected to:'"
proc = Popen(cassandra_cli_cmd, shell=True, stdout=PIPE, stderr=PIPE)
(output, errout) = proc.communicate()
if proc.returncode != 0:
self.fail_status_bits |= self.FAIL_STATUS_SERVER_PORT
else:
self.fail_status_bits &= ~self.FAIL_STATUS_SERVER_PORT
self.send_nodemgr_process_status()
def database_periodic(self):
from database.sandesh.database.ttypes import \
DatabaseUsageStats, DatabaseUsageInfo, DatabaseUsage
(linux_dist, x, y) = platform.linux_distribution()
if (linux_dist == 'Ubuntu'):
(disk_space_used, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2 \`/ContrailAnalytics | grep %` && echo $3 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(disk_space_available, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics | grep %` && echo $4 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(analytics_db_size, error_value) = Popen(
"set `du -skL \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics` && echo $1 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
else:
(disk_space_used, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2 \`/ContrailAnalytics | grep %` && echo $3 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(disk_space_available, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics | grep %` && echo $4 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(analytics_db_size, error_value) = Popen(
"set `du -skL \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics` && echo $1 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
try:
db_stat.disk_space_used_1k = int(disk_space_used)
db_stat.disk_space_available_1k = int(disk_space_available)
db_stat.analytics_db_size_1k = int(analytics_db_size)
except ValueError:
sys.stderr.write("Failed to get database usage" + "\n")
else:
db_info.name = socket.gethostname()
db_info.database_usage = db_stat
db_info.database_usage_stats = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
cassandra_cli_cmd = "cassandra-cli --host " + self.hostip + " --batch < /dev/null | grep 'Connected to:'"
proc = Popen(cassandra_cli_cmd, shell=True, stdout=PIPE, stderr=PIPE)
(output, errout) = proc.communicate()
if proc.returncode != 0:
self.fail_status_bits |= self.FAIL_STATUS_SERVER_PORT
else:
self.fail_status_bits &= ~self.FAIL_STATUS_SERVER_PORT
self.send_nodemgr_process_status()
def database_periodic(self):
try:
cassandra_data_dir = self._get_cassandra_config_option(
"data_file_directories")
analytics_dir = cassandra_data_dir + '/ContrailAnalytics'
if os.path.exists(analytics_dir):
popen_cmd = "set `df -Pk " + analytics_dir + " | grep % | awk '{s+=$3}END{print s}'` && echo $1"
(disk_space_used, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
popen_cmd = "set `df -Pk " + analytics_dir + " | grep % | awk '{s+=$4}END{print s}'` && echo $1"
(disk_space_available, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
popen_cmd = "set `du -skL " + analytics_dir + " | awk '{s+=$1}END{print s}'` && echo $1"
(analytics_db_size, error_value) = \
Popen(popen_cmd, shell=True, stdout=PIPE).communicate()
self.fail_status_bits &= ~self.FAIL_STATUS_DISK_SPACE_NA
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(disk_space_used)
db_stat.disk_space_available_1k = int(disk_space_available)
db_stat.analytics_db_size_1k = int(analytics_db_size)
db_info.name = socket.gethostname()
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
elif 'analytics' not in self.contrail_databases:
self.fail_status_bits &= ~self.FAIL_STATUS_DISK_SPACE_NA
else:
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
except:
sys.stderr.write("Failed to get database usage" + "\n")
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
cassandra_cli_cmd = "cassandra-cli --host " + self.hostip + \
" --batch < /dev/null | grep 'Connected to:'"
proc = Popen(cassandra_cli_cmd, shell=True, stdout=PIPE, stderr=PIPE)
(output, errout) = proc.communicate()
if proc.returncode != 0:
self.fail_status_bits |= self.FAIL_STATUS_SERVER_PORT
else:
self.fail_status_bits &= ~self.FAIL_STATUS_SERVER_PORT
self.send_nodemgr_process_status()
# Record cluster status and shut down cassandra if needed
subprocess.Popen([
"contrail-cassandra-status", "--log-file",
"/var/log/cassandra/status.log", "--debug"
])
def send_database_usage(self):
from database.sandesh.database.ttypes import \
DatabaseUsageStats, DatabaseUsageInfo, DatabaseUsage
(linux_dist, x, y) = platform.linux_distribution()
if (linux_dist == 'Ubuntu'):
(disk_space_used, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2 \`/ContrailAnalytics | grep %` && echo $3 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(disk_space_available, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics | grep %` && echo $4 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(analytics_db_size, error_value) = Popen(
"set `du -skL \`grep -A 1 'data_file_directories:' /etc/cassandra/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics` && echo $1 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
else:
(disk_space_used, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2 \`/ContrailAnalytics | grep %` && echo $3 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(disk_space_available, error_value) = Popen(
"set `df -Pk \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics | grep %` && echo $4 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
(analytics_db_size, error_value) = Popen(
"set `du -skL \`grep -A 1 'data_file_directories:' /etc/cassandra/conf/cassandra.yaml | grep '-' | cut -d'-' -f2\`/ContrailAnalytics` && echo $1 | cut -d'%' -f1",
shell=True,
stdout=PIPE).communicate()
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
try:
db_stat.disk_space_used_1k = int(disk_space_used)
db_stat.disk_space_available_1k = int(disk_space_available)
db_stat.analytics_db_size_1k = int(analytics_db_size)
except ValueError:
sys.stderr.write("Failed to get database usage" + "\n")
else:
db_info.name = socket.gethostname()
db_info.database_usage = db_stat
db_info.database_usage_stats = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
def database_periodic(self, event_mgr):
try:
cassandra_data_dirs = self._get_cassandra_config_option(
"data_file_directories")
cassandra_data_dir_exists = False
total_disk_space_used = 0
total_disk_space_available = 0
total_db_size = 0
for cassandra_data_dir in cassandra_data_dirs:
if CassandraManager.cassandra_old():
analytics_dir = cassandra_data_dir + '/ContrailAnalytics'
else:
import glob
all_analytics_dirs = glob.glob(cassandra_data_dir +
'/ContrailAnalyticsCql*')
if all_analytics_dirs:
#for now we assume the partition for all analytics clusters is same
analytics_dir = all_analytics_dirs[0]
if self._db_name == 'analyticsDb' and os.path.exists(
analytics_dir):
cassandra_data_dir_exists = True
msg = "analytics_dir is " + analytics_dir
event_mgr.msg_log(msg, level=SandeshLevel.SYS_DEBUG)
(disk_space_used, disk_space_available) = (
self.disk_space_helper(analytics_dir))
total_disk_space_used += int(disk_space_used)
total_disk_space_available += int(disk_space_available)
du = subprocess.Popen(["du", "-skl", analytics_dir],
stdout=subprocess.PIPE,
close_fds=True)
db_size, directory = du.communicate()[0].split()
total_db_size += int(db_size)
elif os.path.exists(
cassandra_data_dir) and self._db_name == 'configDb':
cassandra_data_dir_exists = True
msg = "cassandra_dir is " + cassandra_data_dir
(disk_space_used, disk_space_available) = (
self.disk_space_helper(cassandra_data_dir))
total_disk_space_used += int(disk_space_used)
total_disk_space_available += int(disk_space_available)
du = subprocess.Popen(["du", "-skl", cassandra_data_dir],
stdout=subprocess.PIPE,
close_fds=True)
db_size, directory = du.communicate()[0].split()
total_db_size += int(db_size)
if cassandra_data_dir_exists == False:
if ((self._db_name == 'analyticsDb'
and 'analytics' not in self.contrail_databases)
or (self._db_name == 'configDb'
and 'config' not in self.contrail_databases)):
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_DISK_SPACE_NA
else:
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
else:
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_DISK_SPACE_NA
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(total_disk_space_used)
db_stat.disk_space_available_1k = int(
total_disk_space_available)
if self._db_name == 'analyticsDb':
db_stat.analytics_db_size_1k = int(total_db_size)
elif self._db_name == 'configDb':
db_stat.config_db_size_1k = int(total_db_size)
db_info.name = socket.gethostname()
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
except:
msg = "Failed to get database usage"
event_mgr.msg_log(msg, level=SandeshLevel.SYS_ERR)
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
cqlsh_cmd = "cqlsh " + self.hostip + " " + self.db_port + " -e quit"
proc = subprocess.Popen(cqlsh_cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
(output, errout) = proc.communicate()
if proc.returncode != 0:
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_SERVER_PORT
else:
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_SERVER_PORT
event_mgr.send_nodemgr_process_status()
# Send cassandra nodetool information
self.send_database_status(event_mgr)
# Record cluster status and shut down cassandra if needed
self.status()
def database_periodic(self):
try:
cassandra_data_dirs = self._get_cassandra_config_option(
"data_file_directories")
cassandra_data_dir_exists = False
total_disk_space_used = 0
total_disk_space_available = 0
total_analytics_db_size = 0
for cassandra_data_dir in cassandra_data_dirs:
if DatabaseEventManager.cassandra_old():
analytics_dir = cassandra_data_dir + '/ContrailAnalytics'
else:
analytics_dir = cassandra_data_dir + '/ContrailAnalyticsCql'
if os.path.exists(analytics_dir):
cassandra_data_dir_exists = True
msg = "analytics_dir is " + analytics_dir
self.msg_log(msg, level=SandeshLevel.SYS_DEBUG)
df = subprocess.Popen(["df", analytics_dir],
stdout=subprocess.PIPE)
output = df.communicate()[0]
device, size, disk_space_used, disk_space_available, \
percent, mountpoint = output.split("\n")[1].split()
total_disk_space_used += int(disk_space_used)
total_disk_space_available += int(disk_space_available)
du = subprocess.Popen(["du", "-skl", analytics_dir],
stdout=subprocess.PIPE)
analytics_db_size, directory = du.communicate()[0].split()
total_analytics_db_size += int(analytics_db_size)
if cassandra_data_dir_exists == False:
if 'analytics' not in self.contrail_databases:
self.fail_status_bits &= ~self.FAIL_STATUS_DISK_SPACE_NA
else:
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
else:
self.fail_status_bits &= ~self.FAIL_STATUS_DISK_SPACE_NA
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(total_disk_space_used)
db_stat.disk_space_available_1k = int(
total_disk_space_available)
db_stat.analytics_db_size_1k = int(total_analytics_db_size)
db_info.name = socket.gethostname()
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
except:
msg = "Failed to get database usage"
self.msg_log(msg, level=SandeshLevel.SYS_ERR)
self.fail_status_bits |= self.FAIL_STATUS_DISK_SPACE_NA
if not self.cassandra_local_running():
self.fail_status_bits |= self.FAIL_STATUS_SERVER_PORT
else:
self.fail_status_bits &= ~self.FAIL_STATUS_SERVER_PORT
self.send_nodemgr_process_status()
# Record cluster status and shut down cassandra if needed
subprocess.Popen([
"contrail-cassandra-status", "--log-file",
"/var/log/cassandra/status.log", "--debug"
])
def database_periodic(self, event_mgr):
try:
cassandra_data_dirs = self._get_cassandra_config_option(
"data_file_directories")
cassandra_data_dir_exists = False
total_disk_space_used = 0
total_disk_space_available = 0
total_db_size = 0
for data_dir in cassandra_data_dirs:
cdir = None
if self._db_owner == 'analytics':
data = self.exec_cmd("ls -1 {}/".format(data_dir))
all_analytics_dirs = [
n.strip() for n in data.split('\n')
if n.startswith("ContrailAnalyticsCql")
]
if all_analytics_dirs:
# for now we assume the partition for all analytics
# clusters are the same
cdir = data_dir + '/' + all_analytics_dirs[0]
elif self._db_owner == 'config':
cdir = data_dir
if not cdir:
continue
cassandra_data_dir_exists = True
msg = "dir for " + self._db_owner + " is " + cdir
event_mgr.msg_log(msg, level=SandeshLevel.SYS_DEBUG)
(disk_space_used,
disk_space_available) = (self.disk_free(cdir))
total_disk_space_used += int(disk_space_used)
total_disk_space_available += int(disk_space_available)
db_size = self.disk_usage(cdir)
total_db_size += int(db_size)
if not cassandra_data_dir_exists:
if self._db_owner not in self.contrail_databases:
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_DISK_SPACE_NA
else:
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
else:
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_DISK_SPACE_NA
disk_space = int(total_disk_space_used) + int(
total_disk_space_available)
if (disk_space / (1024 * 1024) < self.minimum_diskgb):
# TODO: here was a call that stops cassandra's service
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(total_disk_space_used)
db_stat.disk_space_available_1k = int(
total_disk_space_available)
if self._db_owner == 'analytics':
db_stat.analytics_db_size_1k = int(total_db_size)
elif self._db_owner == 'config':
db_stat.config_db_size_1k = int(total_db_size)
db_info.name = self.hostname
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(data=db_info)
usage_stat.send()
except Exception as e:
msg = "Failed to get database usage: " + str(e)
event_mgr.msg_log(msg, level=SandeshLevel.SYS_ERR)
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
# just check connectivity
if (self.db_user and self.db_password):
cqlsh_cmd = "cqlsh -u {} -p {} {} {} -e quit".format(
self.db_user, self.db_password, self.hostip, self.db_port)
else:
cqlsh_cmd = "cqlsh {} {} -e quit".format(self.hostip, self.db_port)
try:
self.exec_cmd(cqlsh_cmd)
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_SERVER_PORT
except:
msg = "Failed to connect to database by CQL: " + str(e)
event_mgr.msg_log(msg, level=SandeshLevel.SYS_ERR)
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_SERVER_PORT
event_mgr.send_nodemgr_process_status()
# Send cassandra nodetool information
self.send_database_status(event_mgr)
# Record cluster status and shut down cassandra if needed
self.status()
def database_periodic(self, event_mgr):
try:
cassandra_data_dirs = self._get_cassandra_config_option(
"data_file_directories")
cassandra_data_dir_exists = False
total_disk_space_used = 0
total_disk_space_available = 0
total_db_size = 0
for cdir in cassandra_data_dirs:
cassandra_data_dir_exists = True
msg = "dir for " + self._db_owner + " is " + cdir
event_mgr.msg_log(msg, level=SandeshLevel.SYS_DEBUG)
(disk_space_used,
disk_space_available) = (self.disk_free(cdir))
total_disk_space_used += int(disk_space_used)
total_disk_space_available += int(disk_space_available)
db_size = self.disk_usage(cdir)
total_db_size += int(db_size)
if not cassandra_data_dir_exists:
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
else:
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_DISK_SPACE_NA
disk_space = int(total_disk_space_used) + int(
total_disk_space_available)
if (disk_space / (1024 * 1024) < self.minimum_diskgb):
# TODO: here was a call that stops cassandra's service
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE
db_stat = DatabaseUsageStats()
db_info = DatabaseUsageInfo()
db_stat.disk_space_used_1k = int(total_disk_space_used)
db_stat.disk_space_available_1k = int(
total_disk_space_available)
if self._db_owner == 'analytics':
db_stat.analytics_db_size_1k = int(total_db_size)
elif self._db_owner == 'config':
db_stat.config_db_size_1k = int(total_db_size)
db_info.name = self.hostname
db_info.database_usage = [db_stat]
usage_stat = DatabaseUsage(table=self.table, data=db_info)
usage_stat.send()
except Exception as e:
msg = "Failed to get database usage: " + str(e)
event_mgr.msg_log(msg, level=SandeshLevel.SYS_ERR)
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_DISK_SPACE_NA
# just check connectivity
cqlsh_cmd = "cqlsh"
if self.db_use_ssl:
cqlsh_cmd += " --ssl"
if (self.db_user and self.db_password):
cqlsh_cmd += " -u {} -p {}".format(self.db_user, self.db_password)
cqlsh_cmd += " {} {} -e quit".format(self.hostip, self.db_port)
try:
self.exec_cmd(cqlsh_cmd)
event_mgr.fail_status_bits &= ~event_mgr.FAIL_STATUS_SERVER_PORT
except Exception as e:
msg = "Failed to connect to database by CQL: " + str(e)
event_mgr.msg_log(msg, level=SandeshLevel.SYS_ERR)
event_mgr.fail_status_bits |= event_mgr.FAIL_STATUS_SERVER_PORT
event_mgr.send_nodemgr_process_status()
# Send cassandra nodetool information
self.send_database_status(event_mgr)
# Record cluster status and shut down cassandra if needed
self.status()