Python shingle示例

示例#1

def _stacktrace_encoder(id, stacktrace):
    encoded_frames = []

    for frame in stacktrace.values:
        encoded = {}
        for (component_id,
             shingle_label), features in frame.encode_for_similarity():
            assert (
                shingle_label == "ident-shingle"
            ), "Frames cannot use anything other than ident shingles for now"

            if not features:
                continue

            assert (
                len(features) == 1 and component_id not in encoded
            ), "Frames cannot use anything other than ident shingles for now"
            encoded[component_id] = features[0]

        if encoded:
            # add frozen dict
            encoded_frames.append(tuple(sorted(encoded.items())))

    if len(encoded_frames) < 2:
        if encoded_frames:
            yield (id, "frames-ident"), encoded_frames
        return

    yield (id, "frames-pairs"), shingle(2, encoded_frames)

示例#2

文件： similarity.py 项目： zanjs/sentry

     8,
     2,
     60 * 60 * 24 * 30,
     3,
 ),
 BidirectionalMapping({
     'exception:message:character-shingles': 'a',
     'exception:stacktrace:application-chunks': 'b',
     'exception:stacktrace:pairs': 'c',
     'message:message:character-shingles': 'd',
 }), {
     'exception:message:character-shingles':
     ExceptionFeature(lambda exception: map(
         serialize_text_shingle,
         shingle(
             13,
             exception.get('value') or '',
         ),
     )),
     'exception:stacktrace:application-chunks':
     ExceptionFeature(
         lambda exception: map(
             lambda frames: FRAME_SEPARATOR.join(
                 map(
                     serialize_frame,
                     frames,
                 ), ),
             get_application_chunks(exception),
         ), ),
     'exception:stacktrace:pairs':
     ExceptionFeature(
         lambda exception: map(

示例#3

文件： __init__.py 项目： getsentry/sentry

     'exception:stacktrace:pairs': 'c',
     'message:message:character-shingles': 'd',
 }),
 {
     'exception:message:character-shingles': ExceptionFeature(
         lambda exception: text_shingle(
             5,
             exception.value,
         ),
     ),
     'exception:stacktrace:application-chunks': ExceptionFeature(
         lambda exception: get_application_chunks(exception),
     ),
     'exception:stacktrace:pairs': ExceptionFeature(
         lambda exception: shingle(
             2,
             exception.stacktrace.frames,
         ),
     ),
     'message:message:character-shingles': MessageFeature(
         lambda message: text_shingle(
             5,
             message.formatted,
         ),
     ),
 },
 expected_extraction_errors=(
     InterfaceDoesNotExist,
 ),
 expected_encoding_errors=(
     FrameEncodingError,
 ),

示例#4

文件： __init__.py 项目： getsentry/sentry

def text_shingle(n, value):
    return itertools.imap(
        u''.join,
        shingle(n, value),
    )

示例#5

文件： __init__.py 项目： rahulchhabria/sentry-fork-rc

    ),
    Encoder({Frame: get_frame_attributes}),
    BidirectionalMapping({
        "exception:message:character-shingles": "a",
        "exception:stacktrace:application-chunks": "b",
        "exception:stacktrace:pairs": "c",
        "message:message:character-shingles": "d",
    }),
    {
        "exception:message:character-shingles":
        ExceptionFeature(lambda exception: text_shingle(5, exception.value)),
        "exception:stacktrace:application-chunks":
        ExceptionFeature(lambda exception: get_application_chunks(exception)),
        "exception:stacktrace:pairs":
        ExceptionFeature(
            lambda exception: shingle(2, exception.stacktrace.frames)),
        "message:message:character-shingles":
        MessageFeature(lambda message: text_shingle(5, message.formatted)),
    },
    expected_extraction_errors=(InterfaceDoesNotExist, ),
    expected_encoding_errors=(FrameEncodingError, ),
)

features2 = GroupingBasedFeatureSet(
    _make_index_backend(
        getattr(settings, "SENTRY_SIMILARITY2_INDEX_REDIS_CLUSTER", None)
        or getattr(settings, "SENTRY_SIMILARITY_INDEX_REDIS_CLUSTER", None)
        or "similarity",
        namespace="sim:2",
    ))

示例#6

文件： __init__.py 项目： rahulchhabria/sentry-fork-rc

def text_shingle(n, value):
    return map("".join, shingle(n, value))

示例#7

文件： similarity.py 项目： duanshuaimin/sentry

     2,
     60 * 60 * 24 * 30,
     3,
 ),
 BidirectionalMapping({
     'exception:message:character-shingles': 'a',
     'exception:stacktrace:application-chunks': 'b',
     'exception:stacktrace:pairs': 'c',
     'message:message:character-shingles': 'd',
 }),
 {
     'exception:message:character-shingles': ExceptionFeature(
         lambda exception: map(
             serialize_text_shingle,
             shingle(
                 13,
                 exception.get('value') or '',
             ),
         )
     ),
     'exception:stacktrace:application-chunks': ExceptionFeature(
         lambda exception: map(
             lambda frames: FRAME_SEPARATOR.join(
                 map(
                     serialize_frame,
                     frames,
                 ),
             ),
             get_application_chunks(exception),
         ),
     ),
     'exception:stacktrace:pairs': ExceptionFeature(

示例#8

def text_shingle(n, value):
    return itertools.imap(u"".join, shingle(n, value))

示例#9

def test_shingle():
    assert list(shingle(5, 'x')) == []
    assert list(shingle(2, ('foo', 'bar', 'baz'))) == [
        ('foo', 'bar'),
        ('bar', 'baz'),
    ]

示例#10

文件： test_iterators.py 项目： duanshuaimin/sentry

def test_shingle():
    assert list(shingle(5, 'x')) == []
    assert list(shingle(2, ('foo', 'bar', 'baz'))) == [
        ('foo', 'bar'),
        ('bar', 'baz'),
    ]

示例#11

def test_shingle():
    assert list(shingle(5, "x")) == []
    assert list(shingle(2, ("foo", "bar", "baz"))) == [("foo", "bar"), ("bar", "baz")]