def test_job_property(thread_local_ret, mocker, monkeypatch):
mock_func = mocker.Mock()
monkeypatch.setattr(
core_transforms.KlioContext, "_create_klio_job_obj", mock_func
)
klio_ns = core_transforms.KlioContext()
if not thread_local_ret:
# sanity check / clear out thread local
klio_ns._thread_local.klio_job = None
else:
klio_ns._thread_local.klio_job = mocker.Mock()
ret_value = klio_ns.job
if not thread_local_ret:
mock_func.assert_called_once_with()
assert (
mock_func.return_value
== ret_value
== klio_ns._thread_local.klio_job
)
else:
mock_func.assert_not_called()
assert klio_ns._thread_local.klio_job == ret_value
def test_metrics_property(thread_local_ret, mocker, monkeypatch):
mock_func = mocker.Mock()
monkeypatch.setattr(
core_transforms.KlioContext, "_get_metrics_registry", mock_func
)
klio_ns = core_transforms.KlioContext()
if not thread_local_ret:
# sanity check / clear out thread local
klio_ns._thread_local.klio_metrics = None
else:
klio_ns._thread_local.klio_metrics = mocker.Mock()
ret_value = klio_ns.metrics
if not thread_local_ret:
mock_func.assert_called_once_with()
assert (
mock_func.return_value
== ret_value
== klio_ns._thread_local.klio_metrics
)
else:
mock_func.assert_not_called()
assert klio_ns._thread_local.klio_metrics == ret_value
def test_load_config_from_file(exists, config_dict, mocker, monkeypatch):
monkeypatch.setattr(os.path, "exists", lambda x: exists)
klio_yaml_file = "/usr/src/config/.effective-klio-job.yaml"
if not exists:
klio_yaml_file = "/usr/lib/python/site-packages/klio/klio-job.yaml"
mock_iglob = mocker.Mock()
mock_iglob.return_value = iter([klio_yaml_file])
monkeypatch.setattr(core_transforms.glob, "iglob", mock_iglob)
open_name = "klio.transforms.core.open"
config_str = yaml.dump(config_dict)
m_open = mocker.mock_open(read_data=config_str)
m = mocker.patch(open_name, m_open)
klio_ns = core_transforms.KlioContext()
klio_config = klio_ns._load_config_from_file()
m.assert_called_once_with(klio_yaml_file, "r")
assert isinstance(klio_config, config.KlioConfig)
if not exists:
mock_iglob.assert_called_once_with(
"/usr/**/klio-job.yaml", recursive=True
)
def test_load_config_from_file_raises(config_dict, mocker, monkeypatch):
monkeypatch.setattr(os.path, "exists", lambda x: False)
mock_iglob = mocker.Mock()
mock_iglob.return_value = iter([])
monkeypatch.setattr(core_transforms.glob, "iglob", mock_iglob)
klio_ns = core_transforms.KlioContext()
with pytest.raises(IOError):
klio_ns._load_config_from_file()
def _klio_context():
if os.getenv("KLIO_DOCS_MODE", "").lower() in ("true", "1"):
# We return a mock object instead of the real thing when building
# Klio's docs. This is because autodocs (which grabs docstrings
# automatically) will import actual objects to document. When
# decorators in particular are imported, they're evaluated. Some
# decorators need access to the Klio context which would otherwise
# fail because building docs does not give access to the
# `KlioConfig` object.
from unittest import mock
return mock.Mock()
return core.KlioContext()
def __init__(cls, name, bases, clsdict):
if not getattr(cls, "_klio", None):
setattr(cls, "_klio", core.KlioContext())
if os.getenv("KLIO_TEST_MODE", "").lower() in ("true", "1"):
return
# TODO: fixme: not every child class will inherit from
# _KlioBaseDataExistenceCheck
if _utils.is_original_process_func(
clsdict, bases, base_class="_KlioBaseDataExistenceCheck"):
setattr(cls, "process", _wrap_process(clsdict["process"]))
cls._klio._transform_name = name
def test_klio_metrics(
runner, metrics_config, exp_clients, klio_config, mocker, monkeypatch,
):
# all should have the native metrics client
exp_clients.append(native_metrics.NativeMetricsClient)
klio_ns = core_transforms.KlioContext()
# sanity check / clear out thread local
klio_ns._thread_local.klio_metrics = None
monkeypatch.setattr(klio_config.pipeline_options, "runner", runner, None)
monkeypatch.setattr(klio_config.job_config, "metrics", metrics_config)
mock_config = mocker.PropertyMock(return_value=klio_config)
monkeypatch.setattr(core_transforms.KlioContext, "config", mock_config)
registry = klio_ns.metrics
assert len(exp_clients) == len(registry._relays)
for actual_relay in registry._relays:
assert any([isinstance(actual_relay, ec) for ec in exp_clients])
if isinstance(actual_relay, logger_metrics.MetricsLoggerClient):
assert actual_relay.disabled is False
def test_klio_metrics(
runner,
metrics_config,
exp_clients,
exp_logger_enabled,
klio_config,
mocker,
monkeypatch,
):
klio_ns = core_transforms.KlioContext()
# sanity check / clear out thread local
klio_ns._thread_local.klio_metrics = None
monkeypatch.setattr(klio_config.pipeline_options, "runner", runner, None)
monkeypatch.setattr(klio_config.job_config, "metrics", metrics_config)
mock_config = mocker.PropertyMock(return_value=klio_config)
monkeypatch.setattr(core_transforms.KlioContext, "config", mock_config)
registry = klio_ns.metrics
for actual_relay in registry._relays:
assert isinstance(actual_relay, exp_clients)
if isinstance(actual_relay, logger_metrics.MetricsLoggerClient):
assert exp_logger_enabled is not actual_relay.disabled
def _klio_context():
return core.KlioContext()
def setup(self):
ctx = core.KlioContext()
self.io_counter = ctx.metrics.counter(f"kmsg-{self.direction}",
transform=self.bind_transform)