def _get_metrics_registry(self):
clients = []
use_logger, use_stackdriver = None, None
metrics_config = self.config.job_config.metrics
# use_logger and use_stackdriver could be False (turn off),
# None (use default config), or a dict of configured values
use_logger = metrics_config.get("logger")
use_stackdriver = metrics_config.get("stackdriver_logger")
# TODO: set runner in OS environment (via klio-exec), since
# the runner defined in config could be overwritten via
# `--direct-runner`.
# i.e.: runner = os.getenv("BEAM_RUNNER", "").lower()
runner = self.config.pipeline_options.runner
if "dataflow" in runner.lower():
# Must explicitly compare to `False` since `None` could be
# the user accepting default config.
# If explicitly false, then just disable logger underneath SD
if use_stackdriver is not False:
sd_client = stackdriver.StackdriverLogMetricsClient(
self.config
)
clients.append(sd_client)
else:
# if use_stackdriver is explicitly false, then make sure
# logger client is disabled since the stackdriver client
# inherits the logger client
use_logger = False
if not len(clients): # setup default client
disabled = False
# User might disable the logger, but we still need a relay
# client if all other relay clients are disabled. This allows
# folks to silence metrics but not need to remove code that
# interacts with `_klio.metrics`.
# Must explicitly compare to `False` since `None` could be
# the user accepting default config
if use_logger is False:
disabled = True
logger_client = metrics_logger.MetricsLoggerClient(
self.config, disabled=disabled
)
clients.append(logger_client)
return metrics_client.MetricsRegistry(
clients, transform_name=self._transform_name
)
def _get_metrics_registry(self):
native_metrics_client = native_metrics.NativeMetricsClient(self.config)
clients = [native_metrics_client]
use_logger, use_shumway = None, None
metrics_config = self.config.job_config.metrics
# use_logger/use_shumway could be False (turn off),
# None (use default config), or a dict of configured values
use_logger = metrics_config.get("logger")
use_shumway = metrics_config.get("shumway")
# TODO: set runner in OS environment (via klio-exec), since
# the runner defined in config could be overwritten via
# `--direct-runner`.
# i.e.: runner = os.getenv("BEAM_RUNNER", "").lower()
runner = self.config.pipeline_options.runner
if kvars.KlioRunner.DIRECT_RUNNER != runner:
if use_logger is None:
use_logger = False
# use shumway when running on DirectGKERunner unless it's explicitly
# turned off/set to False. Don't set it to True if it's set to False
# or it's a dictionary (aka has some configuration)
if kvars.KlioRunner.DIRECT_GKE_RUNNER == runner:
if use_shumway is None:
use_shumway = True
# shumway only works on DirectGKERunner, so we explicitly set it
# to False
else:
use_shumway = False
if use_logger is not False:
logger_client = metrics_logger.MetricsLoggerClient(self.config)
clients.append(logger_client)
if use_shumway is not False:
shumway_client = shumway.ShumwayMetricsClient(self.config)
clients.append(shumway_client)
return metrics_client.MetricsRegistry(
clients, transform_name=self._transform_name)
def test_client_set_timer_unit(metrics_config, exp_timer_unit, klio_config):
klio_config.job_config.metrics = metrics_config
client = logger.MetricsLoggerClient(klio_config)
assert exp_timer_unit == client.timer_unit
def test_client_set_log_level(metrics_config, exp_log_level, klio_config):
klio_config.job_config.metrics = metrics_config
client = logger.MetricsLoggerClient(klio_config)
assert exp_log_level == client.log_level
def client(klio_config):
return logger.MetricsLoggerClient(klio_config)
