def __init__(self, table: Structures.Table, fields: Dict[str, Any], tags: Dict[str, Any] = None,
time_stamp: Union[int, str] = None):
if(not table):
raise ValueError("need table to create query")
if(not fields):
raise ValueError("need at least one value to create query")
if(tags is None):
tags = {}
if(time_stamp is None):
time_stamp = SppUtils.get_actual_time_sec()
# Keyword is always Insert since insert Statement
self.__keyword = Keyword.INSERT
self.__table = table
self.__time_stamp = SppUtils.epoch_time_to_seconds(time_stamp)
fields = self.format_fields(fields)
# make sure you have some fields if they are not privided
if(not list(filter(lambda field_tup: field_tup[1] is not None, fields.items()))):
# need default def to be able to do anything
if(not table.fields):
raise ValueError("fields after formatting empty, need at least one value!")
# only works for strings, any other addition would corrupt the data
for (key, datatype) in table.fields.items():
if(datatype is Structures.Datatype.STRING):
fields[key] = '\"autofilled\"'
break
# test again, improvement possible here
if(not list(filter(lambda field_tup: field_tup[1] is not None, fields.items()))):
raise ValueError("fields after formatting empty, need at least one value!")
self.__fields: Dict[str, Union[int, float, str, bool]] = fields
self.__tags: Dict[str, str] = self.format_tags(tags)
def __insert_metrics_to_buffer(self,
keyword: Keyword,
tables_count: Dict[Table, int],
duration_s: float,
batch_size: int = 1) -> None:
"""Generates statistics per send Batch, total duration is split by item per table.
Arguments:
keyword {Keyword} -- Kind of query.
tables_count {dict} -- Tables send in this batch, key is table, value is count of items.
duration_s {float} -- Time needed to send the batch in seconds.
Keyword Arguments:
batch_size {int} -- Ammount of queries sent in one batch sent at once. (default: {1})
Raises:
ValueError: Any arg does not match the defined parameters or value is unsupported
"""
# Arg checks
if (list(
filter(lambda arg: arg is None,
[keyword, tables_count, duration_s, batch_size]))):
raise ValueError("any metric arg is None. This is not supported")
if (not isinstance(keyword, Keyword)):
raise ValueError("need the keyword to be a instance of keyword.")
if (not tables_count or not isinstance(tables_count, dict)):
raise ValueError(
"need at least one entry of a table in tables_count.")
if (duration_s <= 0):
raise ValueError(
"only positive values are supported for duration. Must be not 0"
)
if (batch_size < 1):
raise ValueError(
"only positive values are supported for batch_size. Must be not 0"
)
# get shared record time to be saved on
querys = []
# save metrics for each involved table individually
for (table, item_count) in tables_count.items():
querys.append(
InsertQuery(
table=self.__metrics_table,
fields={
# Calculating relative duration for this part of whole query
'duration_ms':
duration_s * 1000 * (max(item_count, 1) / batch_size),
'item_count':
item_count,
},
tags={
'keyword': keyword,
'tableName': table.name,
},
time_stamp=SppUtils.get_actual_time_sec()))
self.__insert_buffer[self.__metrics_table] = self.__insert_buffer.get(
self.__metrics_table, []) + querys
def exit(self, error_code: int = SUCCESS_CODE) -> NoReturn:
"""Executes finishing tasks and exits sppmon. To be called every time.
Executes finishing tasks and displays error messages.
Specify only error message if something did went wrong.
Use Error codes specified at top of module.
Does NOT return.
Keyword Arguments:
error {int} -- Errorcode if a error occured. (default: {0})
"""
# error with the command line arguments
# dont store runtime here
if (error_code == ERROR_CODE_CMD_ARGS):
parser.print_help()
sys.exit(ERROR_CODE_CMD_ARGS) # unreachable?
if (error_code == ERROR_CODE_START_ERROR):
ExceptionUtils.error_message(
"Error when starting SPPMon. Please review the errors above")
sys.exit(ERROR_CODE_START_ERROR)
script_end_time = SppUtils.get_actual_time_sec()
LOGGER.debug("Script end time: %d", script_end_time)
try:
if (not self.ignore_setup):
self.store_script_metrics()
if (self.influx_client):
self.influx_client.disconnect()
if (self.rest_client):
self.rest_client.logout()
except ValueError as error:
ExceptionUtils.exception_info(
error=error,
extra_message="Error occured while exiting sppmon")
error_code = ERROR_CODE
self.remove_pid_file()
# Both error-clauses are actually the same, but for possiblility of an split between error cases
# always last due beeing true for any number != 0
if (error_code == ERROR_CODE or error_code):
ExceptionUtils.error_message(
"Error occured while executing sppmon")
elif (not self.ignore_setup):
LOGGER.info("\n\n!!! script completed !!!\n")
print(
f"check log for details: grep \"PID {os.getpid()}\" {self.log_path} > sppmon.log.{os.getpid()}"
)
sys.exit(error_code)
def exit(self, error_code: int = False) -> NoReturn:
"""Executes finishing tasks and exits sppmon. To be called every time.
Executes finishing tasks and displays error messages.
Specify only error message if something did went wrong.
Use Error codes specified at top of module.
Does NOT return.
Keyword Arguments:
error {int} -- Errorcode if a error occured. (default: {False})
"""
# error with the command line arguments
# dont store runtime here
if (error_code == ERROR_CODE_CMD_LINE):
prog_args = []
prog_args.append(sys.argv[0])
prog_args.append("--help")
os.execv(sys.executable, ['python'] + prog_args)
sys.exit(ERROR_CODE_CMD_LINE) # unreachable?
script_end_time = SppUtils.get_actual_time_sec()
LOGGER.debug("Script end time: %d", script_end_time)
try:
if (not self.ignore_setup):
self.store_script_metrics()
if (self.influx_client):
self.influx_client.disconnect()
if (self.rest_client):
self.rest_client.logout()
except ValueError as error:
ExceptionUtils.exception_info(
error=error,
extra_message="Error occured while exiting sppmon")
error_code = ERROR_CODE
if (not error_code):
LOGGER.info("\n\n!!! script completed !!!\n")
self.remove_pid_file()
# Both clauses are actually the same, but for clarification, always last due always beeing true for any number
if (error_code == ERROR_CODE or error_code):
ExceptionUtils.error_message(
"Error occured while executing sppmon")
print(
f"check log for details: grep \"PID {os.getpid()}\" {self.log_path} > sppmon.log.{os.getpid()}"
)
sys.exit(error_code)
def _parse_free_cmd(
ssh_command: SshCommand,
ssh_type: SshTypes) -> Tuple[str, List[Dict[str, Any]]]:
"""Parses the result of the `free` command, splitting it into its parts.
Arguments:
ssh_command {SshCommand} -- command with saved result
ssh_type {SshTypes} -- type of the client
Raises:
ValueError: no command given or no result saved
ValueError: no ssh type given
Returns:
Tuple[str, List[Dict[str, Any]]] -- Tuple of the tablename and a insert list
"""
if (not ssh_command or not ssh_command.result):
raise ValueError("no command given or empty result")
if (not ssh_type):
raise ValueError("no sshtype given")
if (not ssh_command.table_name):
raise ValueError("need table name to insert parsed value")
result_lines = ssh_command.result.splitlines()
header = result_lines[0].split()
header.insert(0, 'name')
values: List[Dict[str, Any]] = list(
map(lambda row: dict(zip(header, row.split())),
result_lines[1:])) # type: ignore
(time_key, _) = SppUtils.get_capture_timestamp_sec()
for row in values:
# remove ':' from name
row['name'] = row['name'][:-1]
# set default needed fields
row['hostName'] = ssh_command.host_name
row['ssh_type'] = ssh_type.name
row[time_key] = SppUtils.get_actual_time_sec()
# recalculate values to be more usefull
if ('available' in row):
row['free'] = int(row.pop('available'))
row['used'] = int(row['total']) - int(row['free'])
return (ssh_command.table_name, values)
def __insert_metrics_to_buffer(self,
keyword: Keyword,
table: Table,
duration_s: float,
item_count: int,
error: Optional[str] = None) -> None:
"""Generates statistics of influx-requests and append them to be sent
Arguments:
keyword {Keyword} -- Kind of query.
tables_count {dict} -- Tables send in this batch, key is table, value is count of items.
duration_s {Optional[float]} -- Time needed to send the batch in seconds. None if a error occured
item_count {int} -- Ammount of queries sent to the server
Keyword Arguments:
error {Optional[str]} -- Error message if an error occured.
Raises:
ValueError: Any arg does not match the defined parameters or value is unsupported
"""
# Arg checks
if (list(
filter(lambda arg: arg is None,
[keyword, table, duration_s, item_count]))):
raise ValueError(
"One of the insert metrics to influx args is None. This is not supported"
)
query = InsertQuery(
table=self.__metrics_table,
fields={
'error': error,
# Calculating relative duration for this part of whole query
'duration_ms': duration_s * 1000,
'item_count': item_count,
},
tags={
'keyword': keyword,
'tableName': table.name,
},
time_stamp=SppUtils.get_actual_time_sec())
old_queries = self.__insert_buffer.get(self.__metrics_table, [])
old_queries.append(query)
self.__insert_buffer[self.__metrics_table] = old_queries
def default_split(
cls, mydict: Dict[str, Any]
) -> Tuple[Dict[str, str], Dict[str, Union[float, int, bool, str]], Union[
str, int, None]]:
"""Do not use this method on purpose! Pre-Defining a table is higly recommended.
Splits the dict into tags/fields/timestamp according to the format of value.
Strings without spaces are tags
Strings with spaces or double-quotations are fields
any number or boolean is a field
if no field is found a dummy field is inserted with a warning and a debug message.
Arguments:
mydict {dict} -- dict with colums as keys. None values are ignored
Raises:
ValueError: Dictonary is not provided or empty
Returns:
(dict, dict, int) -- Tuple of tags, fields and timestamp
"""
if (not mydict):
raise ValueError("at least one entry is required to split")
ExceptionUtils.error_message(
"WARNING: Using default split method, one table is set up only temporary"
)
LOGGER.debug(f"default split args: {mydict}")
# In case only fields are recognized
fields: Dict[str, Union[float, int, bool, str]] = {}
tags: Dict[str, str] = {}
time_stamp: Any = None
for (key, value) in mydict.items():
if (value is None):
# skip if no value
continue
# Check timestamp value if it matches any of predefined time names
if (key in cls.time_key_names):
# self defined logtime has higher priority, only it allows a rewrite
if (not time_stamp or key == 'logTime'):
time_stamp = value
# continue in any case to avoid double insert
continue
if (isinstance(value, (float, int, bool))):
fields[key] = value
# Make a string out of Lists/Dics
if (not isinstance(value, str)):
value = '\"{}\"'.format(value)
if (re.search(r"[\s\[\]\{\}\"]", value)):
fields[key] = value
else:
tags[key] = value
# at least one field is required to insert.
if (not fields):
ExceptionUtils.error_message(f"missing field in {mydict}")
fields["MISSING_FIELD"] = 42
if (time_stamp is None):
ExceptionUtils.error_message(
f"No timestamp value gathered when using default split, using current time: {mydict}"
)
time_stamp = SppUtils.get_actual_time_sec()
return (tags, fields, time_stamp)
def split_by_table_def(
self, mydict: Dict[str, Any]
) -> Tuple[Dict[str, Any], Dict[str, Any], Union[str, int]]:
"""Split the given dict into a pre-defined set of tags, fields and a timestamp.
None-Values and empty strings are ignored.
If there are no fields declared, it will split by a default pattern.
Undeclared collums will produce a warning.
This function uses the tag/field and timestamp definiton declared within this table.
Arguments:
self {Table} -- Table with predefined set of tags and fields
mydict {Dict[str, Any]} -- dict with colums as keys. None-Values are ignored
Raises:
ValueError: If no dict is given or not of type dict.
Returns:
(Dict[str, Any], Dict[str, Any], int) -- Tuple of: tags, fields, timestamp
"""
if (not mydict):
raise ValueError("need at least one value in dict to split")
# if table is not defined use default split
if (not self.fields):
return InfluxUtils.default_split(mydict=mydict)
# fill dicts
# table.fields is a dict, we only need the keys
fields: Dict[str, Any] = dict.fromkeys(self.fields.keys(), None)
tags: Dict[str, Any] = dict.fromkeys(self.tags, None)
# what field should be recorded as time
time_stamp_field = self.time_key
# helper variable to only overwrite if it is not the time_stamp_field
time_overwrite_allowed = True
# actualy timestamp saved
time_stamp: Union[str, int, None] = None
for (key, value) in mydict.items():
# Ignore empty entrys
if (value is None or (isinstance(value, str) and not value)):
continue
# Check timestamp value if it matches any of predefined time names
if (key in time_stamp_field or key in InfluxUtils.time_key_names):
# sppmonCTS has lowest priority, only set if otherwise None
if (time_stamp is None and key == SppUtils.capture_time_key):
time_stamp = value
# time_stamp_field is highest priority. Do not overwrite it.
elif (key is time_stamp_field):
time_overwrite_allowed: bool = False
time_stamp = value
# if time_stamp_field is not used yet, overwrite sppmonCaptureTime or others
elif (time_overwrite_allowed):
time_stamp = value
# if no overwrite allowed, continue and drop field
else:
continue
# Otherwise check for Keys or Fields
if (key in fields):
fields[key] = value
elif (key in tags):
tags[key] = value
elif (key in InfluxUtils.time_key_names
or key in time_stamp_field):
continue
else:
ExceptionUtils.error_message(
f"Not all columns for table {self.name} are declared: {key}"
)
# before key+"MISSING" : Removed to avoid death-circle on repeated queries.
fields[key] = value
if (time_stamp is None):
ExceptionUtils.error_message(
f"No timestamp value gathered for table {self.name}, using current time: {mydict}"
)
time_stamp = SppUtils.get_actual_time_sec()
return (tags, fields, time_stamp)