def test_to_native_string(self):
# type: () -> None
text = u'éâû'
binary = text.encode('utf-8')
if PY3:
assert to_native_string(binary) == text
else:
assert to_native_string(binary) == binary
def __init__(self, check, config, connection_args):
# (MySql, MySQLConfig) -> None
collection_interval = float(
config.statement_metrics_config.get('collection_interval', 10))
if collection_interval <= 0:
collection_interval = 10
super(MySQLStatementMetrics, self).__init__(
check,
rate_limit=1 / float(collection_interval),
run_sync=is_affirmative(
config.statement_metrics_config.get('run_sync', False)),
enabled=is_affirmative(
config.statement_metrics_config.get('enabled', True)),
expected_db_exceptions=(pymysql.err.DatabaseError, ),
min_collection_interval=config.min_collection_interval,
dbms="mysql",
job_name="statement-metrics",
shutdown_callback=self._close_db_conn,
)
self._metric_collection_interval = collection_interval
self._connection_args = connection_args
self._db = None
self._config = config
self.log = get_check_logger()
self._state = StatementMetrics()
self._obfuscate_options = to_native_string(
json.dumps(self._config.obfuscator_options))
# full_statement_text_cache: limit the ingestion rate of full statement text events per query_signature
self._full_statement_text_cache = TTLCache(
maxsize=self._config.full_statement_text_cache_max_size,
ttl=60 * 60 /
self._config.full_statement_text_samples_per_hour_per_query,
) # type: TTLCache
def __init__(self, check, config, shutdown_callback):
collection_interval = float(
config.statement_metrics_config.get('collection_interval', DEFAULT_COLLECTION_INTERVAL)
)
if collection_interval <= 0:
collection_interval = DEFAULT_COLLECTION_INTERVAL
super(PostgresStatementMetrics, self).__init__(
check,
run_sync=is_affirmative(config.statement_metrics_config.get('run_sync', False)),
enabled=is_affirmative(config.statement_metrics_config.get('enabled', True)),
expected_db_exceptions=(psycopg2.errors.DatabaseError,),
min_collection_interval=config.min_collection_interval,
dbms="postgres",
rate_limit=1 / float(collection_interval),
job_name="query-metrics",
shutdown_callback=shutdown_callback,
)
self._metrics_collection_interval = collection_interval
self._config = config
self._state = StatementMetrics()
self._stat_column_cache = []
self._obfuscate_options = to_native_string(json.dumps(self._config.obfuscator_options))
# full_statement_text_cache: limit the ingestion rate of full statement text events per query_signature
self._full_statement_text_cache = TTLCache(
maxsize=config.full_statement_text_cache_max_size,
ttl=60 * 60 / config.full_statement_text_samples_per_hour_per_query,
)
def _strip_whitespace(raw_plan):
tree = ET.fromstring(raw_plan)
for e in tree.iter():
if e.text:
e.text = e.text.strip()
if e.tail:
e.tail = e.tail.strip()
return to_native_string(ET.tostring(tree))
def obfuscate_xml_plan(raw_plan, obfuscator_options=None):
"""
Obfuscates SQL text & Parameters from the provided SQL Server XML Plan
Also strips unnecessary whitespace
"""
tree = ET.fromstring(raw_plan)
for e in tree.iter():
if e.text:
e.text = e.text.strip()
if e.tail:
e.tail = e.tail.strip()
for k in XML_PLAN_OBFUSCATION_ATTRS:
val = e.attrib.get(k, None)
if val:
statement = obfuscate_sql_with_metadata(val, obfuscator_options)
e.attrib[k] = ensure_unicode(statement['query'])
return to_native_string(ET.tostring(tree, encoding="UTF-8"))
def generate_container_profile_config_with_ad(profile):
host = socket.gethostbyname(get_container_ip(SNMP_CONTAINER_NAME))
network = ipaddress.ip_network(u'{}/29'.format(host),
strict=False).with_prefixlen
conf = {
# Make sure the check handles bytes
'network_address': to_native_string(network),
'port': PORT,
'community_string': 'apc_ups',
}
init_config = {}
instance = generate_instance_config([], template=conf)
instance['community_string'] = profile
instance['enforce_mib_constraints'] = False
return {
'init_config': init_config,
'instances': [instance],
}
def _hash_to_hex(hash):
return to_native_string(binascii.hexlify(hash))
def __init__(self, check, config, connection_args):
collection_interval = float(
config.statement_metrics_config.get('collection_interval', 1))
if collection_interval <= 0:
collection_interval = 1
super(MySQLStatementSamples, self).__init__(
check,
rate_limit=1 / collection_interval,
run_sync=is_affirmative(
config.statement_samples_config.get('run_sync', False)),
enabled=is_affirmative(
config.statement_samples_config.get('enabled', True)),
min_collection_interval=config.min_collection_interval,
dbms="mysql",
expected_db_exceptions=(pymysql.err.DatabaseError, ),
job_name="statement-samples",
shutdown_callback=self._close_db_conn,
)
self._config = config
self._version_processed = False
self._connection_args = connection_args
# checkpoint at zero so we pull the whole history table on the first run
self._checkpoint = 0
self._last_check_run = 0
self._db = None
self._configured_collection_interval = self._config.statement_samples_config.get(
'collection_interval', -1)
self._events_statements_row_limit = self._config.statement_samples_config.get(
'events_statements_row_limit', 5000)
self._explain_procedure = self._config.statement_samples_config.get(
'explain_procedure', 'explain_statement')
self._fully_qualified_explain_procedure = self._config.statement_samples_config.get(
'fully_qualified_explain_procedure', 'datadog.explain_statement')
self._events_statements_temp_table = self._config.statement_samples_config.get(
'events_statements_temp_table_name', 'datadog.temp_events')
self._events_statements_enable_procedure = self._config.statement_samples_config.get(
'events_statements_enable_procedure',
'datadog.enable_events_statements_consumers')
self._preferred_events_statements_tables = EVENTS_STATEMENTS_PREFERRED_TABLES
self._has_window_functions = False
events_statements_table = self._config.statement_samples_config.get(
'events_statements_table', None)
if events_statements_table:
if events_statements_table in DEFAULT_EVENTS_STATEMENTS_COLLECTION_INTERVAL:
self._log.debug(
"Configured preferred events_statements_table: %s",
events_statements_table)
self._preferred_events_statements_tables = [
events_statements_table
]
else:
self._log.warning(
"Invalid events_statements_table: %s. Must be one of %s. Falling back to trying all tables.",
events_statements_table,
', '.join(
DEFAULT_EVENTS_STATEMENTS_COLLECTION_INTERVAL.keys()),
)
self._explain_strategies = {
'PROCEDURE': self._run_explain_procedure,
'FQ_PROCEDURE': self._run_fully_qualified_explain_procedure,
'STATEMENT': self._run_explain,
}
self._preferred_explain_strategies = [
'PROCEDURE', 'FQ_PROCEDURE', 'STATEMENT'
]
self._obfuscate_options = to_native_string(
json.dumps(self._config.obfuscator_options))
self._init_caches()
def __init__(self, name, init_config, instances):
super(SQLServer, self).__init__(name, init_config, instances)
self._resolved_hostname = None
self._agent_hostname = None
self.connection = None
self.failed_connections = {}
self.instance_metrics = []
self.instance_per_type_metrics = defaultdict(set)
self.do_check = True
self.tags = self.instance.get("tags", [])
self.reported_hostname = self.instance.get('reported_hostname')
self.autodiscovery = is_affirmative(self.instance.get('database_autodiscovery'))
self.autodiscovery_include = self.instance.get('autodiscovery_include', ['.*'])
self.autodiscovery_exclude = self.instance.get('autodiscovery_exclude', [])
self.autodiscovery_db_service_check = is_affirmative(self.instance.get('autodiscovery_db_service_check', True))
self.min_collection_interval = self.instance.get('min_collection_interval', 15)
self._compile_patterns()
self.autodiscovery_interval = self.instance.get('autodiscovery_interval', DEFAULT_AUTODISCOVERY_INTERVAL)
self.databases = set()
self.ad_last_check = 0
self.proc = self.instance.get('stored_procedure')
self.proc_type_mapping = {'gauge': self.gauge, 'rate': self.rate, 'histogram': self.histogram}
self.custom_metrics = init_config.get('custom_metrics', [])
# DBM
self.dbm_enabled = self.instance.get('dbm', False)
self.statement_metrics_config = self.instance.get('query_metrics', {}) or {}
self.statement_metrics = SqlserverStatementMetrics(self)
self.activity_config = self.instance.get('query_activity', {}) or {}
self.activity = SqlserverActivity(self)
self.cloud_metadata = {}
aws = self.instance.get('aws', {})
gcp = self.instance.get('gcp', {})
azure = self.instance.get('azure', {})
if aws:
self.cloud_metadata.update({'aws': aws})
if gcp:
self.cloud_metadata.update({'gcp': gcp})
if azure:
self.cloud_metadata.update({'azure': azure})
obfuscator_options_config = self.instance.get('obfuscator_options', {}) or {}
self.obfuscator_options = to_native_string(
json.dumps(
{
# Valid values for this can be found at
# https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md#connection-level-attributes
'dbms': 'mssql',
'replace_digits': is_affirmative(
obfuscator_options_config.get(
'replace_digits',
obfuscator_options_config.get('quantize_sql_tables', False),
)
),
'keep_sql_alias': is_affirmative(obfuscator_options_config.get('keep_sql_alias', True)),
'return_json_metadata': is_affirmative(obfuscator_options_config.get('collect_metadata', True)),
'table_names': is_affirmative(obfuscator_options_config.get('collect_tables', True)),
'collect_commands': is_affirmative(obfuscator_options_config.get('collect_commands', True)),
'collect_comments': is_affirmative(obfuscator_options_config.get('collect_comments', True)),
}
)
)
self.static_info_cache = TTLCache(
maxsize=100,
# cache these for a full day
ttl=60 * 60 * 24,
)
# Query declarations
check_queries = []
if is_affirmative(self.instance.get('include_ao_metrics', False)):
check_queries.extend(
[
QUERY_AO_AVAILABILITY_GROUPS,
QUERY_AO_FAILOVER_CLUSTER,
QUERY_AO_FAILOVER_CLUSTER_MEMBER,
]
)
if is_affirmative(self.instance.get('include_fci_metrics', False)):
check_queries.extend([QUERY_FAILOVER_CLUSTER_INSTANCE])
self._check_queries = self._new_query_executor(check_queries)
self.check_initializations.append(self._check_queries.compile_queries)
self.server_state_queries = self._new_query_executor([QUERY_SERVER_STATIC_INFO])
self.check_initializations.append(self.server_state_queries.compile_queries)
# use QueryManager to process custom queries
self._query_manager = QueryManager(
self, self.execute_query_raw, tags=self.tags, hostname=self.resolved_hostname
)
self._dynamic_queries = None
self.check_initializations.append(self.config_checks)
self.check_initializations.append(self._query_manager.compile_queries)
self.check_initializations.append(self.initialize_connection)
def __init__(self, check, config, shutdown_callback):
collection_interval = float(
config.statement_samples_config.get('collection_interval',
DEFAULT_COLLECTION_INTERVAL))
if collection_interval <= 0:
collection_interval = DEFAULT_COLLECTION_INTERVAL
super(PostgresStatementSamples, self).__init__(
check,
rate_limit=1 / collection_interval,
run_sync=is_affirmative(
config.statement_samples_config.get('run_sync', False)),
enabled=is_affirmative(
config.statement_samples_config.get('enabled', True)),
dbms="postgres",
min_collection_interval=config.min_collection_interval,
expected_db_exceptions=(psycopg2.errors.DatabaseError, ),
job_name="query-samples",
shutdown_callback=shutdown_callback,
)
self._check = check
self._config = config
self._tags_no_db = None
self._activity_last_query_start = None
# The value is loaded when connecting to the main database
self._explain_function = config.statement_samples_config.get(
'explain_function', 'datadog.explain_statement')
self._obfuscate_options = to_native_string(
json.dumps(self._config.obfuscator_options))
self._collection_strategy_cache = TTLCache(
maxsize=config.statement_samples_config.get(
'collection_strategy_cache_maxsize', 1000),
ttl=config.statement_samples_config.get(
'collection_strategy_cache_ttl', 300),
)
self._explain_errors_cache = TTLCache(
maxsize=config.statement_samples_config.get(
'explain_errors_cache_maxsize', 5000),
# only try to re-explain invalid statements once per day
ttl=config.statement_samples_config.get('explain_errors_cache_ttl',
24 * 60 * 60),
)
# explained_statements_ratelimiter: limit how often we try to re-explain the same query
self._explained_statements_ratelimiter = RateLimitingTTLCache(
maxsize=int(
config.statement_samples_config.get(
'explained_queries_cache_maxsize', 5000)),
ttl=60 * 60 / int(
config.statement_samples_config.get(
'explained_queries_per_hour_per_query', 60)),
)
# seen_samples_ratelimiter: limit the ingestion rate per (query_signature, plan_signature)
self._seen_samples_ratelimiter = RateLimitingTTLCache(
# assuming ~100 bytes per entry (query & plan signature, key hash, 4 pointers (ordered dict), expiry time)
# total size: 10k * 100 = 1 Mb
maxsize=int(
config.statement_samples_config.get(
'seen_samples_cache_maxsize', 10000)),
ttl=60 * 60 / int(
config.statement_samples_config.get(
'samples_per_hour_per_query', 15)),
)
self._activity_coll_enabled = is_affirmative(
self._config.statement_activity_config.get('enabled', True))
# activity events cannot be reported more often than regular samples
self._activity_coll_interval = max(
self._config.statement_activity_config.get(
'collection_interval', DEFAULT_ACTIVITY_COLLECTION_INTERVAL),
collection_interval,
)
self._activity_max_rows = self._config.statement_activity_config.get(
'payload_row_limit', 3500)
# Keep track of last time we sent an activity event
self._time_since_last_activity_event = 0
self._pg_stat_activity_cols = None
