def __init__(self, instance):
self.log = get_check_logger()
self.host = instance.get('host', instance.get('server', ''))
self.port = int(instance.get('port', 0))
self.tags = list(instance.get('tags', []))
self.mysql_sock = instance.get('sock', '')
self.defaults_file = instance.get('defaults_file', '')
self.user = instance.get('user', '')
self.password = str(instance.get('pass', ''))
self.tags = self._build_tags(instance.get('tags', []))
self.options = instance.get('options', {}) or {
} # options could be None if empty in the YAML
replication_channel = self.options.get('replication_channel')
if replication_channel:
self.tags.append("channel:{0}".format(replication_channel))
self.queries = instance.get('queries', [])
self.ssl = instance.get('ssl', {})
self.connect_timeout = instance.get('connect_timeout', 10)
self.max_custom_queries = instance.get('max_custom_queries',
DEFAULT_MAX_CUSTOM_QUERIES)
self.charset = instance.get('charset')
self.deep_database_monitoring = is_affirmative(
instance.get('deep_database_monitoring', False))
self.statement_metrics_limits = instance.get(
'statement_metrics_limits', None)
self.full_statement_text_cache_max_size = instance.get(
'full_statement_text_cache_max_size', 10000)
self.full_statement_text_samples_per_hour_per_query = instance.get(
'full_statement_text_samples_per_hour_per_query', 1)
self.statement_samples_config = instance.get('statement_samples',
{}) or {}
self.min_collection_interval = instance.get('min_collection_interval',
15)
self.configuration_checks()
def test_get_check_logger_fallback(caplog):
log = get_check_logger()
log.warning("This is a warning")
assert log is DEFAULT_FALLBACK_LOGGER
assert "This is a warning" in caplog.text
def __init__(self, check, config, connection_args):
# (MySql, MySQLConfig) -> None
collection_interval = float(
config.statement_metrics_config.get('collection_interval', 10))
if collection_interval <= 0:
collection_interval = 10
super(MySQLStatementMetrics, self).__init__(
check,
rate_limit=1 / float(collection_interval),
run_sync=is_affirmative(
config.statement_metrics_config.get('run_sync', False)),
enabled=is_affirmative(
config.statement_metrics_config.get('enabled', True)),
expected_db_exceptions=(pymysql.err.DatabaseError, ),
min_collection_interval=config.min_collection_interval,
dbms="mysql",
job_name="statement-metrics",
shutdown_callback=self._close_db_conn,
)
self._metric_collection_interval = collection_interval
self._connection_args = connection_args
self._db = None
self._config = config
self.log = get_check_logger()
self._state = StatementMetrics()
self._obfuscate_options = to_native_string(
json.dumps(self._config.obfuscator_options))
# full_statement_text_cache: limit the ingestion rate of full statement text events per query_signature
self._full_statement_text_cache = TTLCache(
maxsize=self._config.full_statement_text_cache_max_size,
ttl=60 * 60 /
self._config.full_statement_text_samples_per_hour_per_query,
) # type: TTLCache
def __init__(self, check, config):
# (MySql, MySQLConfig) -> None
self._check = check
self._config = config
self._db_hostname = None
self.log = get_check_logger()
self._state = StatementMetrics()
def __init__(
self,
check,
config_host=None,
min_collection_interval=15,
dbms="TODO",
rate_limit=1,
run_sync=False,
enabled=True,
expected_db_exceptions=(),
shutdown_callback=None,
job_name=None,
):
self._check = check
self._config_host = config_host
self._min_collection_interval = min_collection_interval
# map[dbname -> psycopg connection]
self._log = get_check_logger()
self._job_loop_future = None
self._cancel_event = threading.Event()
self._tags = None
self._tags_no_db = None
self._run_sync = None
self._db_hostname = None
self._last_check_run = 0
self._shutdown_callback = shutdown_callback
self._dbms = dbms
self._rate_limiter = ConstantRateLimiter(rate_limit)
self._run_sync = run_sync
self._enabled = enabled
self._expected_db_exceptions = expected_db_exceptions
self._job_name = job_name
def __init__(self, check, config):
self._check = check
self._db = None
self._config = config
self._log = get_check_logger()
self._activity_last_query_start = None
self._last_check_run = 0
self._collection_loop_future = None
self._cancel_event = threading.Event()
self._tags = None
self._tags_str = None
self._service = "postgres"
self._db_hostname = resolve_db_host(self._config.host)
self._enabled = is_affirmative(self._config.statement_samples_config.get('enabled', False))
self._run_sync = is_affirmative(self._config.statement_samples_config.get('run_sync', False))
self._rate_limiter = ConstantRateLimiter(
float(self._config.statement_samples_config.get('collections_per_second', 1))
)
self._explain_function = self._config.statement_samples_config.get(
'explain_function', 'datadog.explain_statement'
)
# explained_statements_cache: limit how often we try to re-explain the same query
self._explained_statements_cache = TTLCache(
maxsize=int(self._config.statement_samples_config.get('explained_statements_cache_maxsize', 5000)),
ttl=60 * 60 / int(self._config.statement_samples_config.get('explained_statements_per_hour_per_query', 60)),
)
# seen_samples_cache: limit the ingestion rate per (query_signature, plan_signature)
self._seen_samples_cache = TTLCache(
# assuming ~100 bytes per entry (query & plan signature, key hash, 4 pointers (ordered dict), expiry time)
# total size: 10k * 100 = 1 Mb
maxsize=int(self._config.statement_samples_config.get('seen_samples_cache_maxsize', 10000)),
ttl=60 * 60 / int(self._config.statement_samples_config.get('samples_per_hour_per_query', 15)),
)
def test_get_check_logger_argument_fallback(caplog):
logger = logging.getLogger()
log = get_check_logger(default_logger=logger)
log.warning("This is a warning")
assert log is logger
assert "This is a warning" in caplog.text
def __init__(self, check, config):
# (MySql, MySQLConfig) -> None
self._check = check
self._config = config
self._db_hostname = None
self.log = get_check_logger()
self._state = StatementMetrics()
# full_statement_text_cache: limit the ingestion rate of full statement text events per query_signature
self._full_statement_text_cache = TTLCache(
maxsize=self._config.full_statement_text_cache_max_size,
ttl=60 * 60 /
self._config.full_statement_text_samples_per_hour_per_query,
) # type: TTLCache
def version_compatible(self, compat_version):
# some patch version numbers contain letters (e.g. 5.0.51a)
# so let's be careful when we compute the version number
log = get_check_logger()
try:
mysql_version = self.version.split('.')
except Exception as e:
log.warning("Cannot compute mysql version, assuming it's older.: %s", e)
return False
log.debug("MySQL version %s", mysql_version)
patchlevel = int(re.match(r"([0-9]+)", mysql_version[2]).group(1))
version = (int(mysql_version[0]), int(mysql_version[1]), patchlevel)
return version >= compat_version
def __init__(self, instance):
self.log = get_check_logger()
self.host = instance.get('host', instance.get('server', ''))
self.port = int(instance.get('port', 0))
self.tags = list(instance.get('tags', []))
self.mysql_sock = instance.get('sock', '')
self.defaults_file = instance.get('defaults_file', '')
self.user = instance.get('user', '')
self.password = str(instance.get('pass', ''))
self.tags = instance.get('tags', [])
self.options = instance.get('options', {}) or {} # options could be None if empty in the YAML
self.queries = instance.get('queries', [])
self.ssl = instance.get('ssl', {})
self.connect_timeout = instance.get('connect_timeout', 10)
self.max_custom_queries = instance.get('max_custom_queries', DEFAULT_MAX_CUSTOM_QUERIES)
self.configuration_checks()
def __init__(self, agent_check):
self.agent_check = agent_check
self.log = get_check_logger()
self.log.debug('Selecting remote connection for method of collection')
self.agent_check._tags.append('server_hostname:{}'.format(
self.agent_check._server_hostname))
self.agent_check._tags.append('server:{}'.format(
self.agent_check._server))
self.agent_check._tags.append('port:{}'.format(self.agent_check._port))
self._fetch_intermediate_certs = is_affirmative(
self.agent_check.instance.get(
'fetch_intermediate_certs',
self.agent_check.init_config.get('fetch_intermediate_certs',
False)))
self._intermediate_cert_refresh_interval = (
# Convert minutes to seconds
float(
self.agent_check.instance.get(
'intermediate_cert_refresh_interval', 60)) * 60)
def __init__(self, init_config, instance_config, service_check_handler):
self.instance = instance_config
self.service_check_handler = service_check_handler
self.log = get_check_logger()
# mapping of raw connections based on conn_key to different databases
self._conns = {}
self.timeout = int(
self.instance.get('command_timeout', self.DEFAULT_COMMAND_TIMEOUT))
self.existing_databases = None
self.server_version = int(
self.instance.get('server_version',
self.DEFAULT_SQLSERVER_VERSION))
self.adoprovider = self.default_adoprovider
self.valid_connectors = []
if adodbapi is not None:
self.valid_connectors.append('adodbapi')
if pyodbc is not None:
self.valid_connectors.append('odbc')
self.default_connector = init_config.get('connector', 'adodbapi')
if self.default_connector.lower() not in self.valid_connectors:
self.log.error(
"Invalid database connector %s, defaulting to adodbapi",
self.default_connector)
self.default_connector = 'adodbapi'
self.connector = self.get_connector()
self.adoprovider = init_config.get('adoprovider',
self.default_adoprovider)
if self.adoprovider.upper() not in self.valid_adoproviders:
self.log.error(
"Invalid ADODB provider string %s, defaulting to %s",
self.adoprovider, self.default_adoprovider)
self.adoprovider = self.default_adoprovider
self.log.debug('Connection initialized.')
def __init__(self, config):
# type: (MySQLConfig) -> None
self.config = config
self.log = get_check_logger()
self._state = StatementMetrics()
def __init__(self, agent_check):
self.agent_check = agent_check
self.log = get_check_logger()
self.log.debug('Selecting local connection for method of collection')
if self.agent_check._tls_validate_hostname and self.agent_check._server_hostname:
self.agent_check._tags.append('server_hostname:{}'.format(self.agent_check._server_hostname))
def __init__(self, instance):
self.log = get_check_logger()
self.host = instance.get('host', instance.get('server', ''))
self.port = int(instance.get('port', 0))
self.reported_hostname = instance.get('reported_hostname', '')
self.tags = list(instance.get('tags', []))
self.mysql_sock = instance.get('sock', '')
self.defaults_file = instance.get('defaults_file', '')
self.user = instance.get('username', instance.get('user', ''))
self.password = str(instance.get('password', instance.get('pass', '')))
self.tags = self._build_tags(instance.get('tags', []))
self.options = instance.get('options', {}) or {
} # options could be None if empty in the YAML
replication_channel = self.options.get('replication_channel')
if replication_channel:
self.tags.append("channel:{0}".format(replication_channel))
self.queries = instance.get('queries', [])
self.ssl = instance.get('ssl', {})
self.additional_status = instance.get('additional_status', [])
self.additional_variable = instance.get('additional_variable', [])
self.connect_timeout = instance.get('connect_timeout', 10)
self.max_custom_queries = instance.get('max_custom_queries',
DEFAULT_MAX_CUSTOM_QUERIES)
self.charset = instance.get('charset')
self.dbm_enabled = is_affirmative(
instance.get('dbm', instance.get('deep_database_monitoring',
False)))
self.statement_metrics_limits = instance.get(
'statement_metrics_limits', None)
self.full_statement_text_cache_max_size = instance.get(
'full_statement_text_cache_max_size', 10000)
self.full_statement_text_samples_per_hour_per_query = instance.get(
'full_statement_text_samples_per_hour_per_query', 1)
self.statement_samples_config = instance.get(
'query_samples', instance.get('statement_samples', {})) or {}
self.statement_metrics_config = instance.get('query_metrics', {}) or {}
self.activity_config = instance.get('query_activity', {}) or {}
self.cloud_metadata = {}
aws = instance.get('aws', {})
gcp = instance.get('gcp', {})
azure = instance.get('azure', {})
if aws:
self.cloud_metadata.update({'aws': aws})
if gcp:
self.cloud_metadata.update({'gcp': gcp})
if azure:
self.cloud_metadata.update({'azure': azure})
self.min_collection_interval = instance.get('min_collection_interval',
15)
self.only_custom_queries = is_affirmative(
instance.get('only_custom_queries', False))
obfuscator_options_config = instance.get('obfuscator_options',
{}) or {}
self.obfuscator_options = {
# Valid values for this can be found at
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md#connection-level-attributes
'dbms':
'mysql',
'replace_digits':
is_affirmative(
obfuscator_options_config.get(
'replace_digits',
obfuscator_options_config.get('quantize_sql_tables',
False))),
'keep_sql_alias':
is_affirmative(
obfuscator_options_config.get('keep_sql_alias', True)),
'return_json_metadata':
is_affirmative(
obfuscator_options_config.get('collect_metadata', True)),
'table_names':
is_affirmative(
obfuscator_options_config.get('collect_tables', True)),
'collect_commands':
is_affirmative(
obfuscator_options_config.get('collect_commands', True)),
'collect_comments':
is_affirmative(
obfuscator_options_config.get('collect_comments', True)),
}
self.configuration_checks()
def __init__(self, check, config, connection_args):
self._check = check
self._version_processed = False
self._connection_args = connection_args
# checkpoint at zero so we pull the whole history table on the first run
self._checkpoint = 0
self._log = get_check_logger()
self._last_check_run = 0
self._db = None
self._tags = None
self._tags_str = None
self._service = "mysql"
self._collection_loop_future = None
self._cancel_event = threading.Event()
self._rate_limiter = ConstantRateLimiter(1)
self._config = config
self._db_hostname = resolve_db_host(self._config.host)
self._enabled = is_affirmative(
self._config.statement_samples_config.get('enabled', False))
self._run_sync = is_affirmative(
self._config.statement_samples_config.get('run_sync', False))
self._collections_per_second = self._config.statement_samples_config.get(
'collections_per_second', -1)
self._events_statements_row_limit = self._config.statement_samples_config.get(
'events_statements_row_limit', 5000)
self._explain_procedure = self._config.statement_samples_config.get(
'explain_procedure', 'explain_statement')
self._fully_qualified_explain_procedure = self._config.statement_samples_config.get(
'fully_qualified_explain_procedure', 'datadog.explain_statement')
self._events_statements_temp_table = self._config.statement_samples_config.get(
'events_statements_temp_table_name', 'datadog.temp_events')
self._events_statements_enable_procedure = self._config.statement_samples_config.get(
'events_statements_enable_procedure',
'datadog.enable_events_statements_consumers')
self._preferred_events_statements_tables = EVENTS_STATEMENTS_PREFERRED_TABLES
self._has_window_functions = False
events_statements_table = self._config.statement_samples_config.get(
'events_statements_table', None)
if events_statements_table:
if events_statements_table in DEFAULT_EVENTS_STATEMENTS_COLLECTIONS_PER_SECOND:
self._log.debug(
"Configured preferred events_statements_table: %s",
events_statements_table)
self._preferred_events_statements_tables = [
events_statements_table
]
else:
self._log.warning(
"Invalid events_statements_table: %s. Must be one of %s. Falling back to trying all tables.",
events_statements_table,
', '.join(DEFAULT_EVENTS_STATEMENTS_COLLECTIONS_PER_SECOND.
keys()),
)
self._explain_strategies = {
'PROCEDURE': self._run_explain_procedure,
'FQ_PROCEDURE': self._run_fully_qualified_explain_procedure,
'STATEMENT': self._run_explain,
}
self._preferred_explain_strategies = [
'PROCEDURE', 'FQ_PROCEDURE', 'STATEMENT'
]
self._init_caches()
self._statement_samples_client = _new_statement_samples_client()
def __init__(self, yamlconfig):
# type: (List[Union[str, Dict]]) -> None
self.log = get_check_logger()
self.config = self._build_relations_config(yamlconfig)
self.has_relations = len(self.config) > 0
def __init__(self, instance):
self.log = get_check_logger()
self.channel = instance.get('channel') # type: str
self.queue_manager_name = instance.get('queue_manager',
'default') # type: str
if not self.channel or not self.queue_manager_name:
msg = "channel, queue_manager are required configurations"
raise ConfigurationError(msg)
host = instance.get('host') # type: str
port = instance.get('port') # type: str
self.connection_name = instance.get('connection_name') # type: str
if (host or port) and self.connection_name:
raise ConfigurationError(
'Specify only one host/port or connection_name configuration, '
'(host={}, port={}, connection_name={}).'.format(
host, port, self.connection_name))
if not self.connection_name:
host = host or 'localhost'
port = port or '1414'
self.connection_name = "{}({})".format(host, port)
self.username = instance.get('username') # type: str
self.password = instance.get('password') # type: str
self.queues = instance.get('queues', []) # type: List[str]
self.queue_patterns = instance.get('queue_patterns',
[]) # type: List[str]
self.queue_regex = [
re.compile(regex) for regex in instance.get('queue_regex', [])
] # type: List[Pattern]
self.auto_discover_queues = is_affirmative(
instance.get('auto_discover_queues', False)) # type: bool
self.collect_statistics_metrics = is_affirmative(
instance.get('collect_statistics_metrics', False)) # type: bool
if int(self.auto_discover_queues) + int(bool(
self.queue_patterns)) + int(bool(self.queue_regex)) > 1:
self.log.warning(
"Configurations auto_discover_queues, queue_patterns and queue_regex are not intended to be used "
"together.")
self.channels = instance.get('channels', []) # type: List[str]
self.channel_status_mapping = self.get_channel_status_mapping(
instance.get('channel_status_mapping')) # type: Dict[str, str]
self.convert_endianness = instance.get('convert_endianness', False)
custom_tags = instance.get('tags', []) # type: List[str]
tags = [
"queue_manager:{}".format(self.queue_manager_name),
"connection_name:{}".format(self.connection_name),
] # type: List[str]
tags.extend(custom_tags)
if host or port:
# 'host' is reserved and 'mq_host' is used instead
tags.extend({"mq_host:{}".format(host), "port:{}".format(port)})
self.tags_no_channel = tags
self.tags = tags + ["channel:{}".format(self.channel)
] # type: List[str]
# SSL options
self.ssl = is_affirmative(instance.get('ssl_auth',
False)) # type: bool
self.ssl_cipher_spec = instance.get(
'ssl_cipher_spec', 'TLS_RSA_WITH_AES_256_CBC_SHA') # type: str
self.ssl_key_repository_location = instance.get(
'ssl_key_repository_location',
'/var/mqm/ssl-db/client/KeyringClient') # type: str
self.ssl_certificate_label = instance.get(
'ssl_certificate_label') # type: str
if instance.get('ssl_auth') is None and (
instance.get('ssl_cipher_spec')
or instance.get('ssl_key_repository_location')
or self.ssl_certificate_label):
self.log.info(
"ssl_auth has not been explictly enabled but other SSL options have been provided. "
"SSL will be used for connecting")
self.ssl = True
self.mq_installation_dir = instance.get('mq_installation_dir',
'/opt/mqm/')
self._queue_tag_re = instance.get('queue_tag_re',
{}) # type: Dict[str, str]
self.queue_tag_re = self._compile_tag_re()
raw_mqcd_version = instance.get('mqcd_version', 6)
try:
self.mqcd_version = getattr(
pymqi.CMQC,
'MQCD_VERSION_{}'.format(raw_mqcd_version)) # type: int
except (ValueError, AttributeError):
raise ConfigurationError(
"mqcd_version must be a number between 1 and 9. {} found.".
format(raw_mqcd_version))
self.instance_creation_datetime = dt.datetime.now(UTC)
def __init__(self):
self.log = get_check_logger()
def __init__(self, config):
self.config = config
self.log = get_check_logger()
self._state = StatementMetrics()
