def startup(self, config=None):
try:
self._config = config
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def collect(self, collection_interval):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called collect() in Logstash Plugin process")
base_values = super(LogstashPlugin, self).\
collect(collection_interval)
base_values['Hostname'] = self._local_hostname
if self._host_shortid:
base_values['HostShortID'] = self._host_shortid
if self._nuocaCollectionName:
base_values['nuocaCollectionName'] = self._nuocaCollectionName
rval = []
collection_count = len(self._logstash_collect_queue)
if not collection_count:
return rval
for i in range(collection_count):
collected_dict = self._logstash_collect_queue.pop(0)
collected_dict.update(base_values)
if 'timestamp' in collected_dict:
dt = date_parse(collected_dict['timestamp'])
epoch_seconds = timegm(dt.timetuple())
epoch_millis = epoch_seconds * 1000 + dt.microsecond / 1000
collected_dict['TimeStamp'] = epoch_millis
rval.append(collected_dict)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def startup(self, config=None):
try:
self._config = config
# Validate the configuration.
required_config_items = [
'admin_host', 'domain_username', 'domain_password'
]
if not self.has_required_config_items(config,
required_config_items):
return False
# Don't reveal the domain password in the NuoCA log file.
display_config = {}
display_config.update(config)
display_config['domain_password'] = ''
nuoca_log(
logging.INFO,
"NuoAdminMonitor plugin config: %s" % str(display_config))
self._admin_host = os.path.expandvars(config['admin_host'])
self._domain_username = os.path.expandvars(
config['domain_username'])
self._domain_password = os.path.expandvars(
config['domain_password'])
if 'admin_collect_interval' in config:
self._admin_collect_interval = config['admin_collect_interval']
if 'admin_collect_timeout' in config:
self._admin_collect_interval = config['admin_collect_timeout']
if 'host_uuid_shortname' in config:
self._host_uuid_shortname = config['host_uuid_shortname']
if 'admin_rest_api_port' in config:
if isinstance(config['admin_rest_api_port'], int):
self._admin_rest_api_port = config['admin_rest_api_port']
else:
self._admin_rest_api_port = int(
os.path.expandvars(config['admin_rest_api_port']))
self._base_url = "http://%s:%d/api/2" % (self._admin_host,
self._admin_rest_api_port)
self._domain_enforcer_url = "%s/domain/enforcer" % self._base_url
self._regions_url = "%s/regions" % self._base_url
self._auth = HTTPBasicAuth(self._domain_username,
self._domain_password)
nuoca_start_ts = None
if 'nuoca_start_ts' in config:
nuoca_start_ts = config['nuoca_start_ts']
self._admin_collect_sync = IntervalSync(
self._admin_collect_interval, seed_ts=nuoca_start_ts)
self._enabled = True
self._timer_thrd = threading.Thread(target=self._timer_thread)
self._timer_thrd.daemon = True
self._timer_thrd.start()
return True
except Exception as e:
nuoca_log(logging.ERROR, "NuoAdminMonitor Plugin: %s" % str(e))
return False
def store(self, ts_values):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called store() in MPCounterPlugin process")
rval = super(PrinterPlugin, self).store(ts_values)
print(ts_values)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def store(self, ts_values):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called store() in MPClientOutputPlugin process")
rval = super(RestClientOutputPlugin, self).store(ts_values)
requests.post(self._config["url"], json=json.dumps(ts_values))
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def startup(self, config=None):
try:
self._config = config
required_config_items = ['url']
if not self.has_required_config_items(config,
required_config_items):
return False
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
def startup(self, config=None):
try:
self._config = config
if config:
if 'increment' in config:
self._increment_value = int(config['increment'])
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def store(self, ts_values):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called store() in MPCounterPlugin process")
rval = super(FilePlugin, self).store(ts_values)
json.dump(ts_values, self._fp)
self._fp.write("\n")
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def runTest(self):
nuoca_util.initialize_logger("/tmp/nuoca.test.log")
nuoca_util.nuoca_set_log_level(logging.INFO)
info_message = "Info message"
nuoca_util.nuoca_log(logging.INFO, info_message)
self.assertEquals(info_message,
nuoca_util.nuoca_get_last_log_message())
error_message = "Error message"
nuoca_util.nuoca_log(logging.ERROR, error_message)
self.assertEquals(error_message,
nuoca_util.nuoca_get_last_log_error_message())
nuoca_util.nuoca_logging_shutdown()
def startup(self, config=None):
try:
self._dbt2_log_dir = None
required_config_items = ['dbt2_log_dir']
if not self.has_required_config_items(config,
required_config_items):
return False
self._dbt2_log_dir = config['dbt2_log_dir']
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def collect(self, collection_interval):
rval = None
try:
nuoca_log(logging.DEBUG, "Called collect() in MPCounterPlugin process")
collected_values = super(CounterPlugin, self).collect(collection_interval)
self.increment()
collected_values["counter"] = self.get_count()
rval = []
rval.append(collected_values)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def _run_logstash_thread(self):
self._logstash_subprocess = None
if 'logstashInputFilePath' in self._config:
os.environ["LOGSTASH_INPUT_FILE_PATH"] = \
self._config['logstashInputFilePath']
if self._logstash_sincedb_path:
os.environ["LOGSTASH_SINCEDB_PATH"] = self._logstash_sincedb_path
try:
popen_args = [
self._logstash_bin, '--node.name', self._nuocaCollectionName,
"--path.config", self._logstash_config
]
if self._logstash_options:
popen_args.extend(self._logstash_options)
self._logstash_subprocess = \
subprocess.Popen(popen_args, stdout=subprocess.PIPE)
except Exception as e:
msg = "logstash process: %s" % str(e)
msg += "\nlogstash_bin: %s" % self._logstash_bin
msg += "\nlogstash_config: %s" % self._logstash_config
nuoca_log(logging.ERROR, msg)
return
try:
while self._enabled:
json_object = None
line = self._logstash_subprocess.stdout.readline()
if line:
self._line_counter += 1
try:
json_object = json.loads(line)
except ValueError:
# These messages are typical log messages about logstash itself
# which are written to stdout. Just write them to the nuoca.log
msg = "logstash message: %s" % line
nuoca_log(logging.INFO, msg)
if json_object:
self._logstash_collect_queue.append(json_object)
nuoca_log(
logging.INFO, "Logstash plugin run_logstash_thread "
"completed %s lines" % str(self._line_counter))
except Exception as e:
msg = "logstash process: %s" % str(e)
nuoca_log(logging.ERROR, msg)
pass
try:
if self._logstash_subprocess:
self._logstash_subprocess.kill()
except Exception as e:
msg = "Excpetion trying to kill logstash process: %s" % str(e)
nuoca_log(logging.ERROR, msg)
def startup(self, config=None):
try:
self._config = config
required_config_items = ['filePath']
if not self.has_required_config_items(config,
required_config_items):
return False
self._file_path = os.path.expandvars(config['filePath'])
self._fp = open(self._file_path, 'w')
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def startup(self, config=None):
try:
self._config = config
# Make sure that zabbix agent is running.
zabbix_agentd = search_running_processes('zabbix_agentd')
if not zabbix_agentd:
nuoca_log(logging.ERROR, "Zabbix agent daemon is not running.")
return False
# Make sure that zabbix_get is installed.
command = "which zabbix_get"
(exit_code, stdout, stderr) = execute_command(command)
if exit_code != 0 or 'zabbix_get' not in stdout:
nuoca_log(logging.ERROR, "Cannot locate zabbix_get command.")
return False
# Validate the configuration.
required_config_items = ['autoDiscoverMonitors', 'server', 'keys']
if not self.has_required_config_items(config,
required_config_items):
return False
if config['autoDiscoverMonitors']:
self._auto_discover_monitors()
nuoca_log(logging.INFO,
"ZBX plugin config: %s" % str(self._config))
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def __init__(self, parent_pipe, plugin_name, plugin_type):
"""
:param parent_pipe: Pipe setup by Yaspy
:param plugin_name: Name of the plugin.
:type plugin_name: ``str``
:param plugin_type: Plugin type: one of: "Input", "Output", "Transform"
:type plugin_type: ``str``
"""
nuoca_log(logging.INFO, "Creating plugin: %s" % plugin_name)
self._parent_pipe = parent_pipe
self._plugin_name = plugin_name
self._type = plugin_type
self._enabled = False
IMultiprocessChildPlugin.__init__(self, parent_pipe=parent_pipe)
def startup(self, config=None):
nuoca_log(logging.INFO, "Setting up oltpbenchplugin..")
try:
self._config = config
required_config_items = ['log_dir', 'log_file']
if not self.has_required_config_items(config,
required_config_items):
return False
for var in 'log_file', 'log_dir':
exec("self._" + var + " = str(config['" + var + "'])")
return True
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return False
def startup(self, config=None):
try:
self._config = config
# Validate the configuration.
required_config_items = [
'broker', 'domain_username', 'domain_password'
]
if not self.has_required_config_items(config,
required_config_items):
return False
# Don't reveal the domain password in the NuoCA log file.
display_config = {}
display_config.update(config)
display_config['domain_password'] = ''
nuoca_log(
logging.INFO,
"NuoAdminMonitor plugin config: %s" % str(display_config))
self._broker = os.path.expandvars(config['broker'])
self._domain_username = os.path.expandvars(
config['domain_username'])
self._domain_password = os.path.expandvars(
config['domain_password'])
if 'database_regex_pattern' in config:
self._database_regex_pattern = config['database_regex_pattern']
if 'host_uuid_shortname' in config:
self._host_uuid_shortname = config['host_uuid_shortname']
self._enabled = True
self._domain_metrics = \
get_nuodb_metrics(
self._broker,
self._domain_password,
listener=MetricsProducer,
user=self._domain_username)
self._thread = threading.Thread(target=self._nuomon_handler_thread)
self._thread.daemon = True
self._thread.start()
try_count = 0
while not self._numon_handler_ready and try_count < 5:
try_count += 1
time.sleep(1)
return self._numon_handler_ready
except Exception as e:
nuoca_log(logging.ERROR, "NuoMon Plugin: %s" % str(e))
return False
def process_metrics(self, metrics):
if self.parse_errors > 0:
nuoca_log(
logging.INFO, "DBT2: process_metrics: there were " +
self.parse_errors + " parse errors reading the logs")
duration = float(self.end_time - self.start_time)
if self.transactions > 1 and duration > 0:
lat_avg = self.sum_latencies / self.transactions
metrics['notpm'] = float(self.transactions) * 60.0 / duration
metrics['lat_avg'] = float(lat_avg)
# Estimating 90th percentile from standard deviation
stddev = sqrt((self.sum_latencies_sq / self.transactions) -
(lat_avg * lat_avg))
metrics['lat_90th'] = float(lat_avg + stddev * 1.28)
return metrics
else:
return None
def collect(self, collection_interval):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called collect() in NuoAdminMonitor Plugin process")
base_values = super(NuoAdminMonitorPlugin,
self).collect(collection_interval)
collection_count = len(self.monitor_collect_queue)
if not collection_count:
return rval
rval = []
for i in range(collection_count):
collected_dict = self.monitor_collect_queue.pop(0)
rval.append(collected_dict)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def startup(self, config=None):
try:
self._config = config
required_config_items = ['HOST', 'PORT', 'INDEX']
if not self.has_required_config_items(config,
required_config_items):
return False
self.elastic_hosts = [{
"host": self._config['HOST'],
"port": self._config['PORT']
}]
self.es_obj = Elasticsearch(self.elastic_hosts, timeout=10)
logger = logging.getLogger('elasticsearch')
logger.setLevel(logging.WARNING)
return True
except Exception as e:
nuoca_log(logging.ERROR, "ElasticSearch Store error: %s" % str(e))
return False
def collect(self, collection_interval):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called collect() in OLTPBenchPlugin process...")
collected_values = \
super(OLTPBenchPlugin, self).collect(collection_interval)
rval = []
rval.append(collected_values)
if not self.run_parser():
return None
if self._metrics_dict:
for metric_item in self._metrics_dict:
if self._metrics_dict[metric_item]:
collected_values[metric_item] = \
coerce_numeric(self._metrics_dict[metric_item])
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def run_parser(self):
full_logpath = os.path.join(self._log_dir, self._log_file)
cmd = "printf 'tps=\"'; \
printf $(grep 'Throughput: ' {} | tail -1 | sed 's/.*Throughput: //; s/ Tps//')\\\"; \
printf ' latency_average=\"'; \
printf $(grep 'Latency Average: ' {} | tail -1 | sed 's/.*Latency Average: //; s/ ms//')\\\";".format(
full_logpath, full_logpath)
nuoca_log(logging.DEBUG, "Running command: %s" % (format(cmd)))
try:
output = subprocess.check_output(cmd,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError as e:
nuoca_log(
logging.ERROR,
"run_parser ERROR: RC: %d, ERR: %s" % (e.returncode, e.output))
return None
nuoca_log(logging.DEBUG, "Parser output: %s" % (format(output)))
self._metrics_dict = dict(re.findall(r'(\w+)="([^"]+)"', output))
if self._metrics_dict["tps"] == self._last_tps:
self._metrics_dict["tps"] = None
else:
self._last_tps = self._metrics_dict["tps"]
if self._metrics_dict["latency_average"] == self._last_ltc:
self._metrics_dict["latency_average"] = None
else:
self._last_ltc = self._metrics_dict["latency_average"]
return True
def collect(self, collection_interval):
uuid_hostname_regex = \
'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-'
rval = None
try:
nuoca_log(logging.DEBUG,
"Called collect() in NuoMonitor Plugin process")
base_values = super(NuoMonitorPlugin,
self).collect(collection_interval)
collection_count = len(self._nuomonitor_collect_queue)
if not collection_count:
return rval
rval = []
for i in range(collection_count):
collected_dict = self._nuomonitor_collect_queue.pop(0)
m = re.search(self._database_regex_pattern,
collected_dict['Database'])
if m:
if self._host_uuid_shortname:
m2 = re.search(uuid_hostname_regex,
collected_dict['Hostname'])
if m2:
shortid = collected_dict['Hostname'][37:]
if 'NodeType' in collected_dict:
if collected_dict['NodeType'] == 'Transaction':
shortid += "(TE)"
elif collected_dict['NodeType'] == 'Storage':
shortid += "(SM)"
shortid_with_pid = shortid + str(
collected_dict['ProcessId'])
collected_dict['HostShortID'] = shortid
collected_dict[
'HostShortIDwithPID'] = shortid_with_pid
rval.append(collected_dict)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def has_required_config_items(self, config, required_config_items):
"""
Check that the configuration contains the required items.
:param config: Configuration to check
:type config: ``dict``
:param required_config_items: List of required configuration item names
:type required_config_items: ``List[str]``
:return: False if config or required items are missing. Otherwise
return True.
"""
if not config:
nuoca_log(logging.ERROR,
"%s plugin: missing config file." % self._plugin_name)
return False
for config_item in required_config_items:
if config_item not in config:
nuoca_log(
logging.ERROR, "%s plugin: '%s' missing from config." %
(self._plugin_name, config_item))
return False
return True
def store(self, ts_values):
rval = None
try:
nuoca_log(logging.DEBUG,
"Called store() in MPElasticSearch process")
rval = super(ElasticSearchPlugin, self).store(ts_values)
req_resp = self.es_obj.index(index=self._config['INDEX'],
doc_type='nuoca',
body=ts_values)
nuoca_log(logging.DEBUG,
"ElasticSearch response: %s" % str(req_resp))
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def get_rest_url(self, rest_url):
try:
req = requests.get(rest_url,
auth=self._auth,
timeout=self._admin_collect_timeout)
if req.status_code != 200:
err_msg = "NuoAdminMon: Error code '%d' when " \
"calling Admin Rest API: %s" \
% (req.status_code, rest_url)
nuoca_log(logging.ERROR, err_msg)
return {'nuoca_collection_error': err_msg}
return req.json()
except requests.RequestException as e:
err_msg = str(e)
nuoca_log(logging.ERROR, err_msg)
return {'nuoca_collection_error': err_msg}
except Exception as e:
err_msg = str(e)
nuoca_log(logging.ERROR, err_msg)
return {'nuoca_collection_error': err_msg}
def collect(self, collection_interval):
rval = None
try:
dbt2_parser = Dbt2ResultsParser()
# Find two most recent mix logs
if not os.path.exists(self._dbt2_log_dir):
nuoca_log(
logging.WARNING,
"DBT2 log path not found (" + self._dbt2_log_dir + ").")
else:
entries = (os.path.join(self._dbt2_log_dir, fn)
for fn in os.listdir(self._dbt2_log_dir)
if fn.startswith('mix'))
entries = ((os.stat(path), path) for path in entries)
entries = list(
sorted(((stat[ST_CTIME], path) for stat, path in entries)))
if len(entries) < 2:
nuoca_log(logging.INFO,
"DBT2: There are not two mix logs yet.")
else:
most_recent = entries[-1][1]
second_recent = entries[-2][1]
end_time = time.time()
start_time = end_time - collection_interval
with open(second_recent, 'r') as mixlog:
self.read_chunk(dbt2_parser, mixlog, start_time, False)
self.read_chunk(dbt2_parser, mixlog, end_time, True)
with open(most_recent, 'r') as mixlog:
self.read_chunk(dbt2_parser, mixlog, start_time, False)
self.read_chunk(dbt2_parser, mixlog, end_time, True)
metrics = super(Dbt2Plugin,
self).collect(collection_interval)
metrics = dbt2_parser.process_metrics(metrics)
if metrics is not None:
rval = [metrics]
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def collect(self, collection_interval):
rval = []
collected_values = super(ZabbixPlugin,
self).collect(collection_interval)
try:
for key in self._config['keys']:
command = "zabbix_get -s localhost -k %s" % key
(exit_code, stdout, stderr) = execute_command(command)
if exit_code != 0:
nuoca_log(logging.ERROR,
"Problem with zabbix_get command.")
return False
if stdout.strip() is 'ZBX_NOTSUPPORTED':
nuoca_log(
logging.WARNING,
"zabbix_get on key '%s' returned ZBX_NOTSUPPORTED" %
key)
collected_values[key] = coerce_numeric(stdout.strip())
rval.append(collected_values)
except Exception as e:
nuoca_log(logging.ERROR, str(e))
return rval
def deactivate(self):
nuoca_log(logging.INFO, "Deactivate plugin: %s" % self.name)
super(NuocaMPPlugin, self).deactivate()
self._enabled = False
def activate(self):
nuoca_log(logging.INFO, "Activate plugin: %s" % self.name)
super(NuocaMPPlugin, self).activate()
self._enabled = True
