def generate_graphs_on_all_params(self):
directory = settings.path_to_graphs + self.name_host + '/'
for column in self.list_ds:
self.generate_graph(column, directory)
print_text(settings.local.complete_graphs, THEMES_MESSAGE.SUCCESS)
def load_localization(self, lang):
print_text("Loading localization...")
self.lang = lang
with open(self.path_to_localization) as f:
self.local = json.loads(
json.dumps(json.load(f)[lang]),
object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
print_text(self.local.localization + self.lang)
def parse_ds(self):
list_ds = []
try:
info = rrdtool.info(self.file)
for ds in re.findall(r"ds\[\w+]\.index", r"" + json.dumps(info)):
res = re.sub(r"\w+\[", "", ds)
res = re.sub(r"].index", "", res)
list_ds.append(res)
except Exception as e:
print_text(settings.local.error, THEMES_MESSAGE.ERROR)
print_text(e, THEMES_MESSAGE.ERROR)
return list_ds
def select_rrd_file(self):
print_text(settings.local.menu_select_rrd_file, THEMES_MESSAGE.INFO)
types_input = [
MenuItem(settings.local.search_on_name_of_param,
self.search_rrd_on_param),
MenuItem(settings.local.search_on_name_of_file,
self.search_rrd_on_filename)
]
set_id(types_input)
for item in types_input:
print_text(str(item.id) + str(". " + item.name))
ans = input(settings.local.input)
try:
for item in types_input:
if item.id == ans:
# self.display_list_rrd_files()
item.function()
break
except Exception as e:
print_text(settings.local.error, THEMES_MESSAGE.ERROR)
print_text(e, THEMES_MESSAGE.ERROR)
def display_list_rrd_files(self):
list_headers = [
settings.local.name, settings.local.description,
settings.local.file
]
list_rows = []
for rrd in sorted(self.list_rrd, key=lambda rrd: rrd.name_host):
list_rows.append([
rrd.name_host, rrd.description,
re.findall("[^/]*\w+$", rrd.path_to_database)[0] + "/" +
rrd.file_name
])
table = tabulate(list_rows, headers=list_headers, tablefmt="orgtbl")
filename = "table_description_params_rrd.txt"
with open_file(filename, settings.path_to_description_of_params,
FORMATS_OPEN.WRITE) as the_file:
the_file.write(table)
print_text(settings.local.table_params_out_file +
settings.path_to_description_of_params + filename)
print_text(settings.local.show_table_params, THEMES_MESSAGE.INFO)
# if input(settings.local.input).lower() == settings.local.yes.lower():
print_text(settings.local.list_rrds_table, THEMES_MESSAGE.INFO)
print_text(table)
return table
def csv_export(self):
directory = settings.path_to_csv + self.name_host + '/'
filename = self.file_name + ".csv"
path_csv = directory + filename
res_xport = self.xport()
with open_file(filename, directory, FORMATS_OPEN.WRITE) as the_file:
list_columns = res_xport["meta"]["legend"]
data = res_xport["data"]
start = res_xport["meta"]["start"]
step = res_xport["meta"]["step"]
end = res_xport["meta"]["end"]
i = 0
the_file.write("timestamp,")
the_file.write("time,")
for column in list_columns:
i += 1
line = re.sub("\"", "", column)
if i < len(list_columns):
line += ","
the_file.write(line)
the_file.write("\n")
timestamp = start - step
for row in data:
i = 0
timestamp += step
line = str(timestamp) + ", "
line += self.parse_timestamp(timestamp) + ", "
the_file.write(line)
for column in row:
i += 1
line = str((0 if column is None else column))
if i < len(list_columns):
line += ","
the_file.write(line)
the_file.write("\n")
print_text(path_csv)
print_text(settings.local.export_csv, THEMES_MESSAGE.SUCCESS)
return path_csv
def init(self):
print_text("Loading settings...")
with open(self.path) as f:
data = json.load(f)
self.root = data["default"]["root"]
self.root_resources = self.root + data["default"]["root_resources"]
self.root_classes = self.root + data["default"]["root_classes"]
self.root_modules = self.root + data["default"]["root_modules"]
self.path_to_rrd_database = self.root_resources + data["default"][
"path_to_rrd_database"]
self.path_to_exports = self.root_resources + data["default"][
"path_to_exports"]
self.path_to_correlation = self.path_to_exports + data["default"][
"path_to_correlation"]
self.path_to_correlation_images = self.path_to_correlation + data[
"default"]["path_to_correlation_images"]
self.path_to_correlation_xls = self.path_to_correlation + data[
"default"]["path_to_correlation_xls"]
self.path_to_merges_params = self.path_to_correlation + data[
"default"]["path_to_merges_params"]
self.path_to_csv = self.path_to_exports + data["default"][
"path_to_csv"]
self.path_to_description_of_params = self.path_to_exports + data[
"default"]["path_to_description_of_params"]
self.path_to_params_rrd = self.path_to_exports + data["default"][
"path_to_params_rrd"]
self.path_to_graphs = self.path_to_exports + data["default"][
"path_to_graphs"]
self.path_to_localization = data["default"]["path_to_localization"]
self.start_point = data["default"]["start_point"]
self.lang = data["default"]["lang"]
self.end_point = data["default"]["end_point"]
self.type_command = data["default"]["type_command"]
self.height_graph = data["default"]["height"]
self.width_graph = data["default"]["width"]
self.load_localization(self.lang)
def generate_graph(self, column, directory):
filename = column + ".png"
path_to_save = check_exist_file(filename, directory)
print_text(settings.local.start_generate + path_to_save + "...")
try:
rrdtool.graph(
path_to_save,
"--imgformat", "PNG", "--height", str(self.height), "--width",
str(self.width), "--start", self.start_point, "--end",
self.end_point, "--vertical-label=Up times/day", "DEF:" +
column + "=" + self.file + ":" + column + ":AVERAGE:step=1",
"CDEF:cdef=" + column + ",1,*",
"LINE2:cdef#FF6666:\"" + column + "\"", "AREA:cdef#FF6666")
except Exception as e:
print_text(settings.local.error, THEMES_MESSAGE.ERROR)
print_text(e, THEMES_MESSAGE.ERROR)
print_text(settings.local.complete_graph, THEMES_MESSAGE.SUCCESS)
def csv_concat(self, rrd1, rrd2):
print_text(settings.local.merging_params)
f1 = rrd1.csv_export()
f2 = rrd2.csv_export()
df1 = pd.read_csv(f1)
df2 = pd.read_csv(f2)
keys1 = list(df1)
keys2 = list(df2)
print_text(keys1)
print_text(keys2)
for idx, row in df2.iterrows():
data = df1[df1["timestamp"] == row["timestamp"]]
if data.empty:
print_text(settings.local.data_merge_on_time_empty,
THEMES_MESSAGE.WARNING)
next_idx = len(df1)
for key in keys2:
df1.at[next_idx, key] = df2.at[idx, key]
else:
i = int(data.index[0])
for key in keys2:
if key not in keys1:
df1.at[i, key] = df2.at[idx, key]
directory = settings.path_to_merges_params + rrd1.name_host + '_' + rrd2.name_host + '/'
filename = "merge_" + rrd1.file_name + "_" + rrd2.file_name + ".csv"
check_exist_file(filename, directory)
path = directory + filename
df1.to_csv(path,
index=False,
encoding="utf-8",
quotechar="",
quoting=csv.QUOTE_NONE)
return df1
def search_rrd_on_filename(self):
print_text(settings.local.menu_of_search_rrd_on_filename,
THEMES_MESSAGE.INFO)
param = input(settings.local.input_filename_rrd)
self.selected_rrd = self.search_rrd(param)
if type(self.selected_rrd) is not RRD:
print_text(settings.local.not_found, THEMES_MESSAGE.WARNING)
else:
print_text(settings.local.search_successful,
THEMES_MESSAGE.SUCCESS)
return self.selected_rrd
def check_exist_file(filename, directory=settings.root_resources):
if not os.path.exists(directory):
try:
os.makedirs(directory)
except Exception as e:
print_text("Error: ", THEMES_MESSAGE.ERROR)
print_text(e, THEMES_MESSAGE.ERROR)
path = directory + filename
if not os.path.isfile(path):
with open(path, "w", newline=""):
print_text('File ' + filename + ' created!',
THEMES_MESSAGE.WARNING)
return path
def search_rrd_on_param(self):
print_text(settings.local.menu_of_search_rrd_on_param,
THEMES_MESSAGE.INFO)
param = input(settings.local.input_name_rrd)
is_found = False
for rrd in self.list_rrd:
if rrd.name_host == param:
is_found = True
self.selected_rrd = rrd
break
if not is_found or type(self.selected_rrd) is not RRD:
print_text(settings.local.not_found, THEMES_MESSAGE.WARNING)
else:
print_text(settings.local.search_successful,
THEMES_MESSAGE.SUCCESS)
return self.selected_rrd
def generate_graph_with_all_params(self):
directory = settings.path_to_graphs + self.name_host + '/'
filename = ''
for column in self.list_ds:
filename = "_" + column
filename += ".png"
path_to_save = check_exist_file(filename, directory)
print_text(settings.local.start_generate + path_to_save + "...")
colors = ["#ff0000", "#007fff", "#26d100", "#e0e000"]
count_color = 0
rrd_args = [
path_to_save, "--imgformat", "PNG", "--height",
str(self.height), "--width",
str(self.width), "--start", self.start_point, "--end",
self.end_point, "--vertical-label=Up times/day"
]
for column in self.list_ds:
rrd_args.append("DEF:" + column + "=" + self.file + ":" + column +
":AVERAGE:step=60")
rrd_args.append("CDEF:cdef_" + column + "=" + column + ",1,*")
rrd_args.append("LINE2:cdef_" + column + colors[count_color] +
":\"" + column + "\"")
rrd_args.append("GPRINT:cdef_" + column +
":AVERAGE:\"AVERAGE\:%8.0lf\"")
rrd_args.append("GPRINT:cdef_" + column +
":MAX:\"Max\:%8.0lf \\n\"")
count_color += 1
try:
rrdtool.graph(rrd_args)
print_text(settings.local.complete_graph, THEMES_MESSAGE.SUCCESS)
except Exception as e:
print_text(e, THEMES_MESSAGE.ERROR)
def parse_all_rrd(self):
print_text(settings.local.parsing_ngraph)
directory = settings.path_to_description_of_params
try:
subprocess.Popen([
"perl", settings.root_resources + "parsengraph.pl",
settings.path_to_description_of_params
],
stdout=subprocess.PIPE).wait()
except Exception as e:
print_text(settings.local.parsing_ngraph_failed,
THEMES_MESSAGE.ERROR)
print_text(e, THEMES_MESSAGE.ERROR)
descriptions = []
with open_file("table_description_params_rrd.csv", directory,
"r") as file:
reader = csv.reader(file)
for row in reader:
descriptions.append(row)
print_text(settings.local.loading_rrds)
for root, dirs, files in os.walk(settings.path_to_rrd_database):
for filename in files:
if filename.endswith(".rrd"):
try:
rrdtool.info(root + "/" + filename)
# name = re.findall("[^/]*\w+$", root)[0]
name = ""
description = ""
for item in descriptions:
if filename.lower() in item[2].lower():
name = item[0]
description = item[1]
break
rrd = RRD(name_host=name,
description=description,
path_to_database=root,
file_name=filename,
start_point=settings.start_point,
end_point=settings.end_point,
type_command=settings.type_command,
height=settings.height_graph,
width=settings.width_graph)
list_ds = rrd.parse_ds
self.list_rrd.append(rrd)
for ds in list_ds:
self.list_all_params.append(
str(rrd.name_host) + "." + str(ds))
except Exception as e:
print_text(e, THEMES_MESSAGE.ERROR)
if len(self.list_rrd) > 0:
self.selected_rrd = self.list_rrd[0]
print_text(
settings.local.complete_load_rrds + "(" +
str(len(self.list_rrd)) + ")", THEMES_MESSAGE.SUCCESS)
else:
print_text(settings.local.not_found_rrds, THEMES_MESSAGE.WARNING)
return self.list_all_params
def correlation_rrd_files(self):
# self.display_list_rrd_files()
rrd1 = self.search_rrd(
input(settings.local.input_filename_rrd_number + "1: "))
rrd2 = self.search_rrd(
input(settings.local.input_filename_rrd_number + "2: "))
if type(rrd1) is not RRD or type(rrd2) is not RRD:
print_text(settings.local.incorrect_input, THEMES_MESSAGE.ERROR)
return
data = self.csv_concat(rrd1, rrd2)
print_text(settings.local.table_params, THEMES_MESSAGE.INFO)
print_text(data.to_string(max_rows=10))
print_text(settings.local.start_correlation)
corrmat = data.corr()
try:
directory = settings.path_to_correlation_xls + rrd1.name_host + '_' + rrd2.name_host + '/'
filename = "correlation_" + rrd1.file_name + '_' + rrd2.file_name + ".xlsx"
check_exist_file(filename, directory)
path_save = directory + filename
corrmat.to_excel(path_save)
except Exception as e:
print_text(e, THEMES_MESSAGE.ERROR)
# with open(path_save + ".txt", "w") as the_file:
# the_file.write(corrmat.to)
print_text(settings.local.table_correlation, THEMES_MESSAGE.INFO)
print_text(str(corrmat))
f, ax = plt.subplots(figsize=(12, 8))
sns.heatmap(corrmat, cmap="YlGnBu", square=True, annot=True, ax=ax)
directory = settings.path_to_correlation_images + rrd1.name_host + '_' + rrd2.name_host + '/'
filename = "correlation_matrix_type_1_" + rrd1.file_name + '_' + rrd2.file_name + ".png"
check_exist_file(filename, directory)
path_save = directory + filename
plt.savefig(path_save)
plt.show()
plt.clf()
plt.close()
print_text(settings.local.graph_correlation_created + path_save,
THEMES_MESSAGE.SUCCESS)
directory = settings.path_to_correlation_images + rrd1.name_host + '_' + rrd2.name_host + '/'
filename = "correlation_matrix_type_2_" + rrd1.file_name + '_' + rrd2.file_name + ".png"
check_exist_file(filename, directory)
path_save = directory + filename
plt.subplots(figsize=(12, 8))
sns.pairplot(data)
plt.savefig(path_save)
plt.show()
plt.clf()
plt.close()
print_text(settings.local.graph_correlation_created + path_save,
THEMES_MESSAGE.SUCCESS)
print_text(settings.local.complete_correlation, THEMES_MESSAGE.SUCCESS)
def export_params_all_rdd_files_to_csv(self):
for rrd in self.list_rrd:
print_text(rrd.csv_export())
def display_settings(self):
print_text(self.local.main_settings, THEMES_MESSAGE.INFO)
print_text("path = ", self.path)
print_text("root = ", self.root)
print_text("root_resources = ", self.root_resources)
print_text("root_classes = ", self.root_classes)
print_text("root_modules = ", self.root_modules)
print_text("path_to_rrd_database = ", self.path_to_rrd_database)
print_text("path_to_exports = ", self.path_to_exports)
print_text("path_to_correlation = ", self.path_to_correlation)
print_text("path_to_correlation_images = ",
self.path_to_correlation_images)
print_text("path_to_correlation_xls = ", self.path_to_correlation_xls)
print_text("path_to_merges_params = ", self.path_to_merges_params)
print_text("path_to_csv = ", self.path_to_csv)
print_text("path_to_description_of_params = ",
self.path_to_description_of_params)
print_text("path_to_params_rrd = ", self.path_to_params_rrd)
print_text("path_to_graphs = ", self.path_to_graphs)
print_text("path_to_localization = ", self.path_to_localization)
print_text("localization = ", self.lang)
print_text("start_point = ", self.start_point)
print_text("end_point = ", self.end_point)
print_text("type_command = ", self.type_command)
print_text("height_graph = ", self.height_graph)
print_text("width_graph = ", self.width_graph)
def display_info(self):
print_text(settings.local.info_rrd, THEMES_MESSAGE.INFO)
print_text(self.file)
print_text(rrdtool.info(self.file))
def display_params(self):
print_text(settings.local.params_rrd, THEMES_MESSAGE.INFO)
for key, value in self.__dict__.items():
print_text([key, " =", value])