Пример #1
0
def test_scribe_tail_log_everything():
    env = "fake_env"
    stream_name = "fake_stream"
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    components = ["build", "deploy"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    queue = Queue()
    filter_fn = mock.Mock(return_value=True)

    tailer = iter(
        [
            format_log_line(levels[0], clusters, instance, "build", "level: first. component: build."),
            format_log_line(levels[1], clusters, instance, "deploy", "level: second. component: deploy."),
        ]
    )
    with contextlib.nested(mock.patch("paasta_tools.cli.cmds.logs.scribereader", autospec=True)) as (
        mock_scribereader,
    ):
        mock_scribereader.get_env_scribe_host.return_value = {"host": "fake_host", "port": "fake_port"}
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(env, stream_name, service, levels, components, clusters, queue, filter_fn)
        assert mock_scribereader.get_env_scribe_host.call_count == 1
        mock_scribereader.get_stream_tailer.assert_called_once_with(stream_name, "fake_host", "fake_port")
        assert queue.qsize() == 2
        # Sadly, fetching with a timeout seems to be needed with
        # multiprocessing.Queue (this was not the case with Queue.Queue). It
        # failed 8/10 times with a get_nowait() vs 0/10 times with a 0.1s
        # timeout.
        first_line = queue.get(True, 0.1)
        assert "level: first. component: build." in first_line
        second_line = queue.get(True, 0.1)
        assert "level: second. component: deploy." in second_line
Пример #2
0
def test_format_log_line_rejects_invalid_components():
    with raises(utils.NoSuchLogComponent):
        utils.format_log_line(
            level="debug",
            cluster="fake_cluster",
            service="fake_service",
            instance="fake_instance",
            line="fake_line",
            component="BOGUS_COMPONENT",
        )
Пример #3
0
def test_format_log_line_rejects_invalid_components():
    with raises(utils.NoSuchLogComponent):
        utils.format_log_line(
            level='debug',
            cluster='fake_cluster',
            service='fake_service',
            instance='fake_instance',
            line='fake_line',
            component='BOGUS_COMPONENT',
        )
Пример #4
0
def test_scribe_tail_log_nothing():
    env = 'fake_env'
    stream_name = 'fake_stream'
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    components = ['build', 'deploy']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    queue = Queue()
    filter_fn = mock.Mock(return_value=False)

    tailer = iter([
        format_log_line(
            levels[0],
            clusters,
            instance,
            'build',
            'level: first. component: build.',
        ),
        format_log_line(
            levels[1],
            clusters,
            instance,
            'deploy',
            'level: second. component: deploy.',
        ),
    ])
    with contextlib.nested(
        mock.patch('paasta_tools.cli.cmds.logs.scribereader', autospec=True),
    ) as (
        mock_scribereader,
    ):
        mock_scribereader.get_env_scribe_host.return_value = {
            'host': 'fake_host',
            'port': 'fake_port',
        }
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(
            env,
            stream_name,
            service,
            levels,
            components,
            clusters,
            queue,
            filter_fn,
        )
        assert queue.qsize() == 0
Пример #5
0
def test_format_log_line():
    input_line = 'foo'
    fake_cluster = 'fake_cluster'
    fake_service = 'fake_service'
    fake_instance = 'fake_instance'
    fake_component = 'build'
    fake_level = 'debug'
    fake_now = 'fake_now'
    expected = json.dumps({
        'timestamp': fake_now,
        'level': fake_level,
        'cluster': fake_cluster,
        'service': fake_service,
        'instance': fake_instance,
        'component': fake_component,
        'message': input_line,
    }, sort_keys=True)
    with mock.patch('paasta_tools.utils._now', autospec=True) as mock_now:
        mock_now.return_value = fake_now
        actual = utils.format_log_line(
            level=fake_level,
            cluster=fake_cluster,
            service=fake_service,
            instance=fake_instance,
            component=fake_component,
            line=input_line,
        )
        assert actual == expected
Пример #6
0
def test_format_log_line():
    input_line = "foo"
    fake_cluster = "fake_cluster"
    fake_service = "fake_service"
    fake_instance = "fake_instance"
    fake_component = "build"
    fake_level = "debug"
    fake_now = "fake_now"
    expected = json.dumps(
        {
            "timestamp": fake_now,
            "level": fake_level,
            "cluster": fake_cluster,
            "service": fake_service,
            "instance": fake_instance,
            "component": fake_component,
            "message": input_line,
        },
        sort_keys=True,
    )
    with mock.patch("paasta_tools.utils._now", autospec=True) as mock_now:
        mock_now.return_value = fake_now
        actual = utils.format_log_line(
            level=fake_level,
            cluster=fake_cluster,
            service=fake_service,
            instance=fake_instance,
            component=fake_component,
            line=input_line,
        )
        assert actual == expected
Пример #7
0
def test_format_log_line_with_timestamp():
    input_line = 'foo'
    fake_cluster = 'fake_cluster'
    fake_service = 'fake_service'
    fake_instance = 'fake_instance'
    fake_component = 'build'
    fake_level = 'debug'
    fake_timestamp = 'fake_timestamp'
    expected = json.dumps({
        'timestamp': fake_timestamp,
        'level': fake_level,
        'cluster': fake_cluster,
        'service': fake_service,
        'instance': fake_instance,
        'component': fake_component,
        'message': input_line,
    }, sort_keys=True)
    actual = utils.format_log_line(
        fake_level,
        fake_cluster,
        fake_service,
        fake_instance,
        fake_component,
        input_line,
        timestamp=fake_timestamp
    )
    assert actual == expected
Пример #8
0
def test_paasta_log_line_passes_filter_true_when_default_cluster():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], ANY_CLUSTER, service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is True
Пример #9
0
def test_paasta_log_line_passes_filter_false_when_wrong_level():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line('BOGUS_LEVEL', clusters[0], service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is False
Пример #10
0
def test_paasta_log_line_passes_filter_false_when_wrong_level():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line("BOGUS_LEVEL", clusters[0], service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is False
Пример #11
0
def test_paasta_log_line_passes_filter_true_when_default_cluster():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(levels[0], ANY_CLUSTER, service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is True
Пример #12
0
def test_paasta_log_line_passes_filter_true():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    instances = [instance]
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters, instances) is True
Пример #13
0
def test_paasta_log_line_passes_filter_false_when_wrong_cluster():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(levels[0], 'BOGUS_CLUSTER', service, instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is False
Пример #14
0
def test_chronos_log_line_passes_filter_true_when_service_name_in_string():
    service = "fake_service"
    levels = []
    components = []
    clusters = []
    line = format_log_line(
        "fake_level", clusters, service, "fake_instance", "chronos", "fake message with service name %s" % service
    )
    with mock.patch("paasta_tools.chronos_tools.compose_job_id", autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert logs.chronos_log_line_passes_filter(line, levels, service, components, clusters)
Пример #15
0
def test_paasta_log_line_passes_filter_false_when_wrong_component():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    components = ["build", "deploy"]
    line = "fake_line"
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, "monitoring", line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is False
Пример #16
0
def test_paasta_log_line_passes_filter_false_when_wrong_component():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, 'monitoring', line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters) is False
Пример #17
0
def test_scribe_tail_log_nothing():
    env = "fake_env"
    stream_name = "fake_stream"
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    components = ["build", "deploy"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    queue = Queue()
    filter_fn = mock.Mock(return_value=False)

    tailer = iter(
        [
            format_log_line(
                levels[0],
                clusters,
                instance,
                "build",
                "level: first. component: build.",
            ),
            format_log_line(
                levels[1],
                clusters,
                instance,
                "deploy",
                "level: second. component: deploy.",
            ),
        ]
    )
    with mock.patch(
        "paasta_tools.cli.cmds.logs.scribereader", autospec=True
    ) as mock_scribereader:
        mock_scribereader.get_env_scribe_host.return_value = {
            "host": "fake_host",
            "port": "fake_port",
        }
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(
            env, stream_name, service, levels, components, clusters, queue, filter_fn
        )
        assert queue.qsize() == 0
Пример #18
0
def test_marathon_log_line_passes_filter_false_when_service_name_missing():
    service = "fake_service"
    levels = []
    components = []
    clusters = []
    line = format_log_line(
        "fake_level", clusters, service, "fake_instance", "marathon", "fake message without service name"
    )

    with mock.patch("paasta_tools.cli.cmds.logs.format_job_id", autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert not logs.marathon_log_line_passes_filter(line, levels, service, components, clusters)
Пример #19
0
def parse_marathon_log_line(line, clusters):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ""
    else:
        return format_log_line(
            level="event",
            cluster=clusters[0],
            instance="ALL",
            component="marathon",
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #20
0
Файл: logs.py Проект: ese/paasta
def parse_chronos_log_line(line, clusters):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ''
    else:
        return format_log_line(
            level='event',
            cluster=clusters[0],
            instance='ALL',
            component='chronos',
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #21
0
def test_paasta_log_line_passes_filter_true():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(levels[0], clusters[0], service, instance,
                                     components[0], line)
    assert (logs.paasta_log_line_passes_filter(
        formatted_line, levels, service, components, clusters, instances) is
            True)
Пример #22
0
def parse_chronos_log_line(line, clusters):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ''
    else:
        return format_log_line(
            level='event',
            cluster=clusters[0],
            instance='ALL',
            component='chronos',
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #23
0
def test_paasta_log_line_passes_filter_false_when_wrong_level():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line("BOGUS_LEVEL", clusters[0], service,
                                     instance, components[0], line)
    assert (logs.paasta_log_line_passes_filter(
        formatted_line, levels, service, components, clusters, instances) is
            False)
Пример #24
0
def test_paasta_log_line_passes_filter_true():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    instances = [instance]
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance,
                                     components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service,
                                              components, clusters,
                                              instances) is True
Пример #25
0
def test_paasta_log_line_passes_filter_false_when_wrong_level():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    instances = [instance]
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line('BOGUS_LEVEL', clusters[0], service,
                                     instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service,
                                              components, clusters,
                                              instances) is False
Пример #26
0
def test_paasta_log_line_passes_filter_false_when_invalid_time():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, components[0], line,
                                     timestamp=isodate.datetime_isoformat(datetime.datetime.utcnow()))

    start_time, end_time = logs.generate_start_end_time(from_string="5m", to_string="3m")

    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters,
                                              start_time=start_time, end_time=end_time) is False
Пример #27
0
def parse_marathon_log_line(line, clusters, service):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ''
    else:
        return format_log_line(
            level='event',
            cluster=clusters[0],
            service=service,
            instance='ALL',
            component='marathon',
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #28
0
def parse_chronos_log_line(line, clusters, service):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ""
    else:
        return format_log_line(
            level="event",
            cluster=clusters[0],
            service=service,
            instance="ALL",
            component="chronos",
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #29
0
def parse_chronos_log_line(line, clusters, service):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ""
    else:
        return format_log_line(
            level="event",
            cluster=clusters[0],
            service=service,
            instance="ALL",
            component="chronos",
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #30
0
def parse_marathon_log_line(line, clusters, service):
    utc_timestamp = extract_utc_timestamp_from_log_line(line)
    if not utc_timestamp:
        return ''
    else:
        return format_log_line(
            level='event',
            cluster=clusters[0],
            service=service,
            instance='ALL',
            component='marathon',
            line=line.strip(),
            timestamp=utc_timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f"),
        )
Пример #31
0
def test_paasta_log_line_passes_filter_false_when_invalid_time():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, components[0], line,
                                     timestamp=isodate.datetime_isoformat(datetime.datetime.utcnow()))

    start_time, end_time = logs.generate_start_end_time(from_string="5m", to_string="3m")

    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters,
                                              start_time=start_time, end_time=end_time) is False
Пример #32
0
def test_paasta_log_line_passes_filter_true_when_valid_time():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, components[0], line,
                                     timestamp="2016-06-07T23:46:03+00:00")

    start_time = isodate.parse_datetime("2016-06-07T23:40:03+00:00")
    end_time = isodate.parse_datetime("2016-06-07T23:50:03+00:00")

    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters,
                                              start_time=start_time, end_time=end_time) is True
Пример #33
0
def test_chronos_log_line_passes_filter_true_when_service_name_in_string():
    service = 'fake_service'
    levels = []
    components = []
    clusters = []
    line = format_log_line(
        'fake_level',
        clusters,
        'fake_instance',
        'chronos',
        'fake message with service name %s' % service,
    )
    with mock.patch('paasta_tools.chronos_tools.compose_job_id', autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert logs.chronos_log_line_passes_filter(line, levels, service, components, clusters)
Пример #34
0
def test_paasta_log_line_passes_filter_false_when_wrong_instance():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'non-existant_instance'
    instances = ['fake_instance']
    components = ['build', 'deploy']
    line = 'fake_line'
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(levels[0], 'BOGUS_CLUSTER', service,
                                     instance, components[0], line)
    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service,
                                              components, clusters,
                                              instances) is False
Пример #35
0
def test_paasta_log_line_passes_filter_true_when_valid_time():
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    components = ['build', 'deploy']
    line = 'fake_line'
    formatted_line = format_log_line(levels[0], clusters[0], service, instance, components[0], line,
                                     timestamp="2016-06-07T23:46:03+00:00")

    start_time = isodate.parse_datetime("2016-06-07T23:40:03+00:00")
    end_time = isodate.parse_datetime("2016-06-07T23:50:03+00:00")

    assert logs.paasta_log_line_passes_filter(formatted_line, levels, service, components, clusters,
                                              start_time=start_time, end_time=end_time) is True
Пример #36
0
def test_paasta_log_line_passes_filter_false_when_wrong_instance():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "non-existant_instance"
    instances = ["fake_instance"]
    components = ["build", "deploy"]
    line = "fake_line"
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(levels[0], "BOGUS_CLUSTER", service,
                                     instance, components[0], line)
    assert (logs.paasta_log_line_passes_filter(
        formatted_line, levels, service, components, clusters, instances) is
            False)
Пример #37
0
def test_chronos_log_line_passes_filter_true_when_service_name_in_string():
    service = 'fake_service'
    levels = []
    components = []
    clusters = []
    line = format_log_line(
        'fake_level',
        clusters,
        'fake_instance',
        'chronos',
        'fake message with service name %s' % service,
    )
    with mock.patch('paasta_tools.chronos_tools.compose_job_id', autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert logs.chronos_log_line_passes_filter(line, levels, service, components, clusters)
Пример #38
0
def test_scribe_tail_log_nothing():
    env = "fake_env"
    stream_name = "fake_stream"
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    components = ["build", "deploy"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    queue = Queue()
    filter_fn = mock.Mock(return_value=False)

    tailer = iter(
        [
            format_log_line(levels[0], clusters, instance, "build", "level: first. component: build."),
            format_log_line(levels[1], clusters, instance, "deploy", "level: second. component: deploy."),
        ]
    )
    with contextlib.nested(mock.patch("paasta_tools.cli.cmds.logs.scribereader", autospec=True)) as (
        mock_scribereader,
    ):
        mock_scribereader.get_env_scribe_host.return_value = {"host": "fake_host", "port": "fake_port"}
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(env, stream_name, service, levels, components, clusters, queue, filter_fn)
        assert queue.qsize() == 0
Пример #39
0
def test_marathon_log_line_passes_filter_false_when_service_name_missing():
    service = 'fake_service'
    levels = []
    components = []
    clusters = []
    line = format_log_line(
        'fake_level',
        clusters,
        'fake_instance',
        'marathon',
        'fake message without service name',
    )

    with mock.patch('paasta_tools.cli.cmds.logs.format_job_id', autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert not logs.marathon_log_line_passes_filter(line, levels, service, components, clusters)
Пример #40
0
def test_marathon_log_line_passes_filter_false_when_service_name_missing():
    service = 'fake_service'
    levels = []
    components = []
    clusters = []
    instances = []
    line = format_log_line(
        'fake_level',
        clusters,
        service,
        'fake_instance',
        'marathon',
        'fake message without service name',
    )

    with mock.patch('paasta_tools.cli.cmds.logs.format_job_id', autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert not logs.marathon_log_line_passes_filter(line, levels, service, components, clusters, instances)
Пример #41
0
def test_format_log_line():
    input_line = 'foo'
    fake_cluster = 'fake_cluster'
    fake_instance = 'fake_instance'
    fake_component = 'build'
    fake_level = 'debug'
    fake_now = 'fake_now'
    expected = json.dumps({
        'timestamp': fake_now,
        'level': fake_level,
        'cluster': fake_cluster,
        'instance': fake_instance,
        'component': fake_component,
        'message': input_line,
    }, sort_keys=True)
    with mock.patch('paasta_tools.utils._now', autospec=True) as mock_now:
        mock_now.return_value = fake_now
        actual = utils.format_log_line(fake_level, fake_cluster, fake_instance, fake_component, input_line)
        assert actual == expected
Пример #42
0
def test_paasta_log_line_passes_filter_true_when_default_instance():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = None
    pods = None
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(
        levels[0], ANY_CLUSTER, service, instance, components[0], line,
    )

    assert (
        logs.paasta_log_line_passes_filter(
            formatted_line, levels, service, components, clusters, instances, pods
        )
        is True
    )
Пример #43
0
def test_chronos_log_line_passes_filter_false_when_service_name_missing():
    service = "fake_service"
    levels = []
    components = []
    clusters = []
    instances = []
    line = format_log_line(
        "fake_level",
        clusters,
        service,
        "fake_instance",
        "chronos",
        "fake message without service name",
    )

    with mock.patch("paasta_tools.chronos_tools.compose_job_id",
                    autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert not logs.chronos_log_line_passes_filter(
            line, levels, service, components, clusters, instances)
Пример #44
0
def test_marathon_log_line_passes_filter_true_when_service_name_in_string():
    service = "fake_service"
    levels = []
    components = []
    clusters = []
    instances = []
    line = format_log_line(
        "fake_level",
        clusters,
        service,
        "fake_instance",
        "marathon",
        "fake message with service name %s" % service,
    )
    with mock.patch("paasta_tools.cli.cmds.logs.format_job_id",
                    autospec=True) as format_job_id_patch:
        format_job_id_patch.return_value = service
        assert logs.marathon_log_line_passes_filter(line, levels, service,
                                                    components, clusters,
                                                    instances)
Пример #45
0
def test_paasta_log_line_passes_filter_false_when_wrong_component():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    pods = None
    components = ["build", "deploy"]
    line = "fake_line"
    # component must be legit as well as not in the list of requested
    # components
    formatted_line = format_log_line(
        levels[0], clusters[0], service, instance, "monitoring", line,
    )

    assert (
        logs.paasta_log_line_passes_filter(
            formatted_line, levels, service, components, clusters, instances, pods
        )
        is False
    )
Пример #46
0
def test_paasta_log_line_passes_filter_false_when_invalid_time():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    pods = None
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(
        levels[0],
        clusters[0],
        service,
        instance,
        components[0],
        line,
        timestamp=isodate.datetime_isoformat(datetime.datetime.utcnow()),
    )

    start_time, end_time = logs.generate_start_end_time(
        from_string="5m", to_string="3m"
    )

    assert (
        logs.paasta_log_line_passes_filter(
            formatted_line,
            levels,
            service,
            components,
            clusters,
            instances,
            pods,
            start_time=start_time,
            end_time=end_time,
        )
        is False
    )
Пример #47
0
def test_format_log_line_with_timestamp():
    input_line = "foo"
    fake_cluster = "fake_cluster"
    fake_service = "fake_service"
    fake_instance = "fake_instance"
    fake_component = "build"
    fake_level = "debug"
    fake_timestamp = "fake_timestamp"
    expected = json.dumps(
        {
            "timestamp": fake_timestamp,
            "level": fake_level,
            "cluster": fake_cluster,
            "service": fake_service,
            "instance": fake_instance,
            "component": fake_component,
            "message": input_line,
        },
        sort_keys=True,
    )
    actual = utils.format_log_line(
        fake_level, fake_cluster, fake_service, fake_instance, fake_component, input_line, timestamp=fake_timestamp
    )
    assert actual == expected
Пример #48
0
def test_paasta_log_line_passes_filter_true_when_valid_time():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    pods = None
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(
        levels[0],
        clusters[0],
        service,
        instance,
        components[0],
        line,
        timestamp="2016-06-07T23:46:03+00:00",
    )

    start_time = isodate.parse_datetime("2016-06-07T23:40:03+00:00")
    end_time = isodate.parse_datetime("2016-06-07T23:50:03+00:00")

    assert (
        logs.paasta_log_line_passes_filter(
            formatted_line,
            levels,
            service,
            components,
            clusters,
            instances,
            pods,
            start_time=start_time,
            end_time=end_time,
        )
        is True
    )
Пример #49
0
def test_format_log_line_with_timestamp():
    input_line = 'foo'
    fake_cluster = 'fake_cluster'
    fake_instance = 'fake_instance'
    fake_component = 'build'
    fake_level = 'debug'
    fake_timestamp = 'fake_timestamp'
    expected = json.dumps({
        'timestamp': fake_timestamp,
        'level': fake_level,
        'cluster': fake_cluster,
        'instance': fake_instance,
        'component': fake_component,
        'message': input_line,
    }, sort_keys=True)
    actual = utils.format_log_line(
        fake_level,
        fake_cluster,
        fake_instance,
        fake_component,
        input_line,
        timestamp=fake_timestamp
    )
    assert actual == expected
Пример #50
0
def test_format_log_line_rejects_invalid_components():
    with raises(utils.NoSuchLogComponent):
        utils.format_log_line('fake_service', 'fake_line', 'BOGUS_COMPONENT', 'debug', 'fake_input')
Пример #51
0
def test_scribe_tail_log_everything():
    env = 'fake_env'
    stream_name = 'fake_stream'
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    components = ['build', 'deploy']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    queue = Queue()
    filter_fn = mock.Mock(return_value=True)

    tailer = iter([
        format_log_line(
            levels[0],
            clusters,
            instance,
            'build',
            'level: first. component: build.',
        ),
        format_log_line(
            levels[1],
            clusters,
            instance,
            'deploy',
            'level: second. component: deploy.',
        ),
    ])
    with contextlib.nested(
        mock.patch('paasta_tools.cli.cmds.logs.scribereader', autospec=True),
    ) as (
        mock_scribereader,
    ):
        mock_scribereader.get_env_scribe_host.return_value = {
            'host': 'fake_host',
            'port': 'fake_port',
        }
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(
            env,
            stream_name,
            service,
            levels,
            components,
            clusters,
            queue,
            filter_fn
        )
        assert mock_scribereader.get_env_scribe_host.call_count == 1
        mock_scribereader.get_stream_tailer.assert_called_once_with(
            stream_name,
            'fake_host',
            'fake_port',
        )
        assert queue.qsize() == 2
        # Sadly, fetching with a timeout seems to be needed with
        # multiprocessing.Queue (this was not the case with Queue.Queue). It
        # failed 8/10 times with a get_nowait() vs 0/10 times with a 0.1s
        # timeout.
        first_line = queue.get(True, 0.1)
        assert 'level: first. component: build.' in first_line
        second_line = queue.get(True, 0.1)
        assert 'level: second. component: deploy.' in second_line
Пример #52
0
def test_scribe_tail_log_everything():
    env = 'fake_env'
    stream_name = 'fake_stream'
    service = 'fake_service'
    levels = ['fake_level1', 'fake_level2']
    components = ['build', 'deploy']
    clusters = ['fake_cluster1', 'fake_cluster2']
    instance = 'fake_instance'
    queue = Queue()
    filter_fn = mock.Mock(return_value=True)

    tailer = iter([
        format_log_line(
            levels[0],
            clusters,
            instance,
            'build',
            'level: first. component: build.',
        ),
        format_log_line(
            levels[1],
            clusters,
            instance,
            'deploy',
            'level: second. component: deploy.',
        ),
    ])
    with contextlib.nested(
        mock.patch('paasta_tools.cli.cmds.logs.scribereader', autospec=True),
    ) as (
        mock_scribereader,
    ):
        mock_scribereader.get_env_scribe_host.return_value = {
            'host': 'fake_host',
            'port': 'fake_port',
        }
        mock_scribereader.get_stream_tailer.return_value = tailer
        logs.scribe_tail(
            env,
            stream_name,
            service,
            levels,
            components,
            clusters,
            queue,
            filter_fn
        )
        assert mock_scribereader.get_env_scribe_host.call_count == 1
        mock_scribereader.get_stream_tailer.assert_called_once_with(
            stream_name,
            'fake_host',
            'fake_port',
        )
        assert queue.qsize() == 2
        # Sadly, fetching with a timeout seems to be needed with
        # multiprocessing.Queue (this was not the case with Queue.Queue). It
        # failed 8/10 times with a get_nowait() vs 0/10 times with a 0.1s
        # timeout.
        first_line = queue.get(True, 0.1)
        assert 'level: first. component: build.' in first_line
        second_line = queue.get(True, 0.1)
        assert 'level: second. component: deploy.' in second_line