def get_log_generator(
self,
query_body: dict = None,
index='_all',
scroll='1m',
filters: List[Callable[[LogEntry], bool]] = None
) -> Generator[LogEntry, None, None]:
"""
A generator that yields LogEntry objects constructed from Kubernetes resource logs.
Logs to be returned are defined by passed query and filtered according to passed
filter functions, which have to accept LogEntry as argument and return a boolean value.
:param query_body: ES search query
:param index: ElasticSearch index from which logs will be retrieved, defaults to all indices
:param scroll: ElasticSearch scroll lifetime
:param filters: List of filter functions with signatures f(LogEntry) -> Bool
:return: Generator yielding LogEntry (date, log_content, pod_name, namespace) named tuples.
"""
for log in elasticsearch.helpers.scan(self,
query=query_body,
index=index,
scroll=scroll,
size=1000,
preserve_order=True,
clear_scroll=False):
log_entry = LogEntry(
date=log['_source']['@timestamp'],
content=log['_source']['log'],
pod_name=log['_source']['kubernetes']['pod_name'],
namespace=log['_source']['kubernetes']['namespace_name'])
if not filters or all(f(log_entry) for f in filters):
yield log_entry
def test_filter_log_by_pod_ids():
pod_id = 'test-pod'
log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
pod_name='test-pod',
namespace='default',
content='bla')
assert filter_log_by_pod_ids(pod_ids={pod_id}, log_entry=log_entry) == True
assert filter_log_by_pod_ids(pod_ids={'another-pod-id'},
log_entry=log_entry) == False
def test_filter_log_by_pod_status(status, mocker):
log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
pod_name='test-pod',
namespace='default',
content='bla')
mocked_get_pod_status = mocker.patch(
'logs_aggregator.log_filters.cached_pod_status')
mocked_get_pod_status.return_value = status
assert filter_log_by_pod_status(log_entry, status) == True
#
from click.testing import CliRunner
from unittest.mock import patch, mock_open
import pytest
from commands.experiment import logs
from logs_aggregator.k8s_log_entry import LogEntry
from util.exceptions import K8sProxyOpenError, K8sProxyCloseError
from platform_resources.run import Run
from cli_text_consts import CmdsCommonTexts as CmdsCommonTexts
TEST_LOG_ENTRIES = [
LogEntry(date='2018-04-17T09:28:39+00:00',
content='Warning: Unable to load '
'/usr/share/zoneinfo/right/Factory as time zone. Skipping it.\n',
pod_name='understood-gnat-mysql-868b556f8f-lwdr9',
namespace='default'),
LogEntry(date='2018-04-17T09:28:49+00:00',
content='MySQL init process done. Ready for start up.\n',
pod_name='understood-gnat-mysql-868b556f8f-lwdr9',
namespace='default')
]
def test_show_logs_success(mocker):
es_client_mock = mocker.patch(
'commands.common.logs_utils.K8sElasticSearchClient')
es_client_instance = es_client_mock.return_value
es_client_instance.get_experiment_logs_generator.return_value = TEST_LOG_ENTRIES
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from logs_aggregator.log_filters import filter_log_by_severity,filter_log_by_pod_status,\
SeverityLevel, filter_log_by_pod_ids
from logs_aggregator.k8s_log_entry import LogEntry
from util.k8s.k8s_info import PodStatus
no_severity_log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
content='bla',
pod_name='elasticsearch-5985cbd9d8-5lf55',
namespace='default')
debug_log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
content='[DEBUG ] bla',
pod_name='elasticsearch-5985cbd9d8-5lf55',
namespace='default')
info_log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
content='[INFO ] bla',
pod_name='elasticsearch-5985cbd9d8-5lf55',
namespace='default')
warning_log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
content='[WARNING ] bla',
pod_name='elasticsearch-5985cbd9d8-5lf55',
namespace='default')
error_log_entry = LogEntry(date='2018-04-19T14:27:46+00:00',
content='[ERROR ] bla',