def _data_handler(self):
logger.debug(
f"{self.__class__.__name__} Started with event {id(self._event)}")
while True:
if self.queue.empty():
if self._event.isSet():
break
else:
continue
item = self.queue.get()
try:
logger.debug(
f"Deque item: '{item}' (Current queue size {self.queue.qsize()})"
)
insert_sql_str, rows = item.sql_data
result = self.execute(
insert_sql_str,
rows) if rows else self.execute(insert_sql_str)
item.result = result
logger.debug("Insert item: {}\n\t{}\n\t{}".format(
type(item).__name__, insert_sql_str,
'\n\t'.join([str(r) for r in rows])))
except Exception as e:
f, l = get_error_info()
logger.error(
f"Unexpected error occurred on {type(item).__name__}: {e}; File: {f}:{l}"
)
else:
logger.debug(
f"Item {type(item).__name__} successfully handled")
logger.debug(f"Background task stopped invoked")
def on_connection(self):
try:
with self._lock:
self.login()
yield self.content_object
except RunnerError as e:
self._session_errors.append(e)
logger.warn(
"Non critical error {name}; Reason: {error} (Attempt {real} from {allowed})"
.format(
name=self.host_alias,
error=e,
real=len(self._session_errors),
allowed=self._fault_tolerance,
))
except Exception as e:
logger.error(
"Critical Error {name}; Reason: {error} (Attempt {real} from {allowed})"
.format(
name=self.host_alias,
error=e,
real=len(self._session_errors),
allowed=self._fault_tolerance,
))
GlobalErrors().append(e)
else:
if len(self._session_errors):
logger.debug(
f"Host '{self}': Runtime errors occurred during tolerance period cleared"
)
self._session_errors.clear()
finally:
self.exit()
def __call__(self, outputs, datetime=None) -> bool:
command_out = outputs.get('stdout', None)
time_output = outputs.get('stderr', None)
rc = outputs.get('rc')
try:
exp_rc = self.options.get('rc', None)
if exp_rc:
if rc not in [int(_rc) for _rc in re.split(r'\s*\|\s*', exp_rc)]:
raise AssertionError(
f"Result return rc {rc} not match expected\nStdOut:\n\t{command_out}\nStdErr:\n\t{time_output}")
data = time_output.split(',')
row_dict = DotDict(**{k: v.replace('%', '') for (k, v) in [entry.split(':', 1) for entry in data]})
for k in row_dict.keys():
if k == 'Command':
continue
row_dict.update({k: float(row_dict[k])})
logger.info(f"Command: {row_dict.get('Command')} [Rc: {row_dict.get('Rc')}]")
row = self.table.template(self.host_id, None, *tuple(list(row_dict.values()) + [-1]))
du = model.data_factory(self.table, row, output=command_out, datetime=datetime)
self.data_handler(du)
return True
except Exception as e:
f, li = get_error_info()
logger.error(f"{self.__class__.__name__}: {e}; File: {f}:{li}")
raise RunnerError(f"{self}", f"{e}; File: {f}:{li}")
def __call__(self, output: dict) -> bool:
out = output.get('stdout', None)
err = output.get('stderr', None)
total_output = f'{out}' if out else ''
total_output += ('\n' if len(total_output) > 0 else '') + (f'{err}' if
err else '')
rc = output.get('rc', -1)
exp_rc = self.options.get('rc', None)
expected = self.options.get('expected', None)
prohibited = self.options.get('prohibited', None)
errors = []
if exp_rc:
if not any(
[int(_rc) == rc for _rc in re.split(r'\s*\|\s*', exp_rc)]):
errors.append(f"Rc [{rc}] not match expected - {exp_rc}")
if expected:
if not any([pattern in total_output for pattern in re.split(r'\s*\|\s*', expected)]) or \
not all([pattern in total_output for pattern in re.split(r'\s*\&\s*', expected)]):
errors.append(
"Output not contain expected pattern [{}]".format(
expected))
if prohibited:
if any([pattern in total_output for pattern in re.split(r'\s*\|\s*', prohibited)]) or \
not all([pattern not in total_output for pattern in re.split(r'\s*\&\s*', prohibited)]):
errors.append("Output contain prohibited pattern [{}]".format(
prohibited))
if len(errors) > 0:
st = 'False'
msg = "\nErrors:\n\t{}\n\tRC: {}\nOutput:\n\t{}".format(
'\n\t'.join(errors), rc,
'\n\t'.join(total_output.splitlines()))
logger.error(msg)
else:
st = 'Pass'
msg = 'Output:\n\t{}'.format('\n\t'.join(
total_output.splitlines()))
output_ref = db.CacheLines().upload(msg)
du = model.data_factory(
self.table,
self.table.template(self.host_id, None,
self.options.get('command'), rc, st,
output_ref))
self.data_handler(du)
if st != 'Pass':
if self._tolerance == -1:
pass
elif self._tolerance_counter == self._tolerance:
raise RunnerError(
f"{self}: Error count reach tolerance ({self._tolerance})",
msg)
else:
self._tolerance_counter += 1
return True if st == 'Pass' else False
def execute(self, sql_text, *rows):
try:
return self._db.execute(sql_text, *rows)
except Exception as e:
logger.error("DB execute error: {}\n{}\n{}".format(
e, sql_text, '\n\t'.join([r for r in rows])))
raise
def __format__(self, format_spec):
old_format = self.__str_format
try:
if format_spec != '':
self.__str_format = "{{:{new_format}}}{{}}".format(
new_format=format_spec)
return str(self)
except Exception as e:
logger.error(e)
finally:
self.__str_format = old_format
def stop(self, timeout=5):
if self._event:
self._event.set()
while len(self._threads) > 0:
th = self._threads.pop(0)
try:
th.join(timeout)
logger.debug(f"Thread '{th.name}' gracefully stopped")
except Exception as e:
logger.error(
f"Thread '{th.name}' gracefully stop failed; Error raised: {e}"
)
def is_continue_expected(self):
if not self._evaluate_tolerance():
self.parameters.event.set()
logger.error(f"Stop requested due of critical error")
return False
if self.parameters.event.isSet():
logger.info(f"Stop requested by external source")
return False
if self._internal_event.isSet():
logger.info(f"Stop requested internally")
return False
return True
def _evaluate_tolerance(self):
if len(self._session_errors) == self._fault_tolerance:
e = PlugInError(
f"{self}",
"PlugIn stop invoked; Errors count arrived to limit ({})".
format(
self.host_alias,
self._fault_tolerance,
), *self._session_errors)
logger.error(f"{e}")
GlobalErrors().append(e)
return False
return True
def __parse(self, bitrate_str: str):
try:
m = BITRATE_REGEX.match(str(bitrate_str))
if m is None:
raise AttributeError(
"Wrong bitrate format ({})".format(bitrate_str))
self.__number = float(m.groups()[0])
self.__format_number()
self.set_format(m.groups()[1])
except Exception as e:
logger.error(
"Cannot parse PacketSize value string '{}' with error: {}".
format(bitrate_str, e))
raise e
def start(self, event=Event()):
if self._db.is_new:
for name, table in TableSchemaService().tables.items():
try:
assert not self._db.table_exist(
table.name), f"Table '{name}' already exists"
self._db.execute(
sql_engine.create_table_sql(table.name, table.fields,
table.foreign_keys))
except AssertionError as e:
logger.warn(f"{e}")
except Exception as e:
logger.error(f"Cannot create table '{name}' -> Error: {e}")
raise
self._event = event
dh = Thread(name='DataHandler', target=self._data_handler, daemon=True)
dh.start()
self._threads.append(dh)
def upload_file_to_portal(link_title, file_path):
if not PORTAL:
return
try:
_, file_name = os.path.split(file_path)
with open(file_path, 'rb') as file_reader:
file_data = file_reader.read()
portal_logger.info(link_title, attachment={
'name': file_name,
'data': file_data,
'mime': 'image/png'
})
return True
except RobotServiceException as e:
logger.error(f"Cannot upload file '{file_path}'; Reason: {e}")
except Exception as e:
logger.error(f"Unexpected error during upload file '{file_path}'; Reason: {e}")
return False
def generate_module_statistics(self, period=None, plugin=None, alias=None, **options):
"""
Generate Chart for present monitor data in visual style
Arguments:
- period:
- plugin:
- alias:
- options:
:Return - html link to chart file
Note: In case report portal used chart files will be uploaded into lunch report (See in `Report Portal integration`)
"""
if not os.path.exists(self._image_path):
os.makedirs(self._image_path, exist_ok=True)
module: HostModule = HostRegistryCache().get_connection(alias)
chart_plugins = module.get_plugin(plugin, **options)
chart_title = self._create_chart_title(period, plugin, f"{module}", **options)
marks = _get_period_marks(period, module.host_id) if period else {}
body_data = []
for plugin in chart_plugins:
for chart in plugin.affiliated_charts():
try:
sql_query = chart.compose_sql_query(host_name=plugin.host_alias, **marks)
logger.debug("{}{}\n{}".format(plugin.type, f'_{period}' if period is not None else '', sql_query))
sql_data = db.DataHandlerService().execute(sql_query)
for picture_name, file_path in generate_charts(chart, sql_data, self._image_path, prefix=chart_title):
relative_image_path = os.path.relpath(file_path, os.path.normpath(
os.path.join(self._output_dir, self._log_path)))
body_data.append((picture_name, relative_image_path))
upload_file_to_portal(picture_name, file_path)
except Exception as e:
logger.error(f"Error: {e}")
html_link_path = create_html(self._output_dir, self._log_path, chart_title, *body_data)
html_link_text = f"Chart for <a href=\"{html_link_path}\">'{chart_title}'</a>"
logger.warn(html_link_text, html=True)
return html_link_text
def generate_chart_data(self, query_results: Iterable[Iterable]) \
-> List[Tuple[str, Iterable, Iterable, Iterable[Iterable]]]:
result = []
for type_ in set([
i[0] for i in query_results if any(
[i[0].startswith(section) for section in self._sections])
]):
try:
data = [i[1:] for i in query_results if i[0] == type_]
x_axes = self.x_axes(data, 1)
y_axes = self.y_axes(data)
data = [i[2:] for i in data]
data = [u[0:len(y_axes)] for u in data]
chart_data = f"{type_}", x_axes, y_axes, data
logger.debug(
"Create chart data: {}\n{}\n{}\n{} entries".format(
type_, x_axes, y_axes, len(data)))
result.append(chart_data)
except Exception as e:
f, l = get_error_info()
logger.error(f"Chart generation error: {e}; File: {f}:{l}")
return result
def __call__(self, output) -> bool:
# table_template = self.table.template
try:
stdout = output.get('stdout')
stderr = output.get('stderr')
rc = output.get('rc')
assert rc == 0, f"Last {self.__class__.__name__} ended with rc: {rc}\n{stderr}"
for atop_portion in [
e.strip() for e in stdout.split('ATOP') if e.strip() != ''
]:
lines = atop_portion.splitlines()
f_line = lines.pop(0)
ts = '_'.join(re.split(
r'\s+', f_line)[2:4]) + f".{datetime.now().strftime('%S')}"
system_portion, process_portion = '\n'.join(lines).split(
'PID', 1)
process_portion = 'PID\t' + process_portion
if ts not in self._ts_cache:
self._ts_cache.append(ts)
self.data_handler(
aTopSystem_DataUnit(self.table['system'], self.host_id,
*system_portion.splitlines()))
if ProcessMonitorRegistry().is_active:
data_portion = self._data_unit_class(
self.table['process'],
self.host_id,
*process_portion.splitlines()[1:],
processes_id=self.id)
self.data_handler(data_portion)
except Exception as e:
f, li = get_error_info()
logger.error(
f"{self.__class__.__name__}: Unexpected error: {type(e).__name__}: {e}; File: {f}:{li}"
)
else:
return True
return False
def time_string_reformat(time_stamp):
try:
return datetime.strptime(time_stamp,
from_format).strftime(to_format)
except Exception as e:
logger.error(f"Cannot convert time string: {time_stamp}")
def _generate_atop_system_level(input_text, columns_template, *defaults):
header_regex = re.compile(r'(.+)\|(.+)\|(.+)\|(.+)\|(.+)\|(.+)\|')
res = []
row_mapping = namedtuple(
'ROW', ('Col1', 'Col2', 'Col3', 'Col4', 'Col5', 'SUB_ID'))
for line in header_regex.findall(input_text):
try:
type_, data_ = aTopParser._normalize_line(*line)
sub_id = type_
pattern = OrderedDict()
if type_ in ('PRC', 'PAG'):
pattern.update(
**{
k: aTopParser.try_time_string_to_secs(v)
for k, v in
[re.split(r'\s+', s.strip(), 2) for s in data_]
})
elif type_ in ['CPU', 'cpu']:
pattern.update(
**{
k: v.replace('%', '')
for k, v in
[re.split(r'\s+', s.strip(), 1) for s in data_]
})
if type_ == 'cpu':
for k, v in pattern.items():
if k.startswith('cpu'):
_cpu_str, _wait = re.split(r'\s+', v, 1)
pattern.pop(k)
pattern.update({'wait': _wait})
sub_id = k.replace('cpu', 'cpu_').upper()
break
type_ = 'CPU'
else:
sub_id = 'CPU_All'
elif type_ == 'CPL':
pattern.update(
**{
k: v
for k, v in
[re.split(r'\s+', s.strip(), 1) for s in data_]
})
elif type_ in ['MEM', 'SWP']:
pattern.update(
**{
k: v
for k, v in
[re.split(r'\s+', s.strip(), 1) for s in data_]
})
for k in pattern.keys():
pattern[k] = Size(pattern[k]).set_format('M').number
elif type_ in ['LVM', 'DSK', 'NET']:
items = [re.split(r'\s+', s.strip()) for s in data_]
for item in items:
if len(item) == 1 or item[1] == '----':
pattern.update({'source': '-1'})
sub_id = f"{type_}_{item[0]}"
elif len(item) >= 2:
pattern.update({item[0]: item[1].replace('%', '')})
else:
pattern.update(
{item[0]: re.sub(r'[\sKbpms%]+', '', item[1])})
else:
raise TypeError(f"Unknown line type: {' '.join(line)}")
pattern.update(SUB_ID=sub_id)
res.append(
columns_template(*[
*defaults, type_,
json.dumps(row_mapping(
*pattern.keys()), indent=True), *pattern.values()
]))
except ValueError as e:
logger.error(f"aTop parse error: {e}")
except Exception as e:
f, l = get_error_info()
logger.error(
"aTop unknown parse error: {}; File: {}:{}\n{}".format(
e, f, l, line))
raise
return res
