Python ngrams Examples

Example #1

File: predicates.py Project: shahin/dedupe

def twoGramFingerprint(field):
    if len(field) > 1:
        return (u''.join(
            sorted(gram.strip()
                   for gram in set(ngrams(field.replace(' ', ''), 2)))), )
    else:
        return ()

Example #2

def commonSixGram(field):
    """Split the field into overlapping windows of 6 characters (spaces removed).

    Examples:
    .. code:: python
        > print(commonFourGram('John Woodward'))
        > {'JohnWo', 'Woodwa', 'hnWood', 'nWoodw', 'odward', 'ohnWoo', 'oodwar'}
    """
    return set(ngrams(field.replace(' ', ''), 6))

Example #3

File: predicates.py Project: zmaril/dedupe

def commonSixGram(field):
    """return 6-grams"""
    return ngrams(field, 6)

Example #4

File: predicates.py Project: zmaril/dedupe

def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)

Example #5

File: predicates.py Project: datamade/dedupe

def commonSixGram(field):
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))

Example #6

def twoGramFingerprint(field):
    return (u''.join(sorted(gram.strip() for gram in ngrams(field, 2))), )

Example #7

File: predicates.py Project: shahin/dedupe

def commonFourGram(field):
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))

Example #8

File: predicates.py Project: shahin/dedupe

def oneGramFingerprint(field):
    return (u''.join(sorted(set(ngrams(field.replace(' ', ''), 1)))).strip(), )

Example #9

File: predicates.py Project: zhenglinyi/dedupe

def commonSixGram(field: str) -> Set[str]:
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))

Example #10

File: predicates.py Project: 01-/dedupe

 def preprocess(self, doc) :
     return tuple(ngrams(doc.replace(' ', ''), 2))

Example #11

File: predicates.py Project: jgoleary/dedupe

def commonSixGram(field):
    """return 6-grams"""
    return ngrams(field, 6)

Example #12

File: predicates.py Project: jgoleary/dedupe

def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)

Example #13

File: predicates.py Project: sql-analytics/dedupe-entries

def twoGramFingerprint(field) :
    return (u''.join(sorted(gram.strip() for gram in ngrams(field, 2))),)

Example #14

File: predicates.py Project: sql-analytics/dedupe-entries

def oneGramFingerprint(field) :
    return (u''.join(sorted(ngrams(field, 1))).strip(),)

Example #15

File: predicates.py Project: zhenglinyi/dedupe

 def preprocess(self, doc):
     return tuple(sorted(ngrams(" ".join(strip_punc(doc).split()), 2)))

Example #16

File: predicates.py Project: zhenglinyi/dedupe

def commonFourGram(field: str) -> Set[str]:
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))

Example #17

File: predicates.py Project: datamade/dedupe

 def preprocess(self, doc):
     return tuple(sorted(ngrams(" ".join(strip_punc(doc).split()), 2)))

Example #18

File: predicates.py Project: shahin/dedupe

 def preprocess(self, doc):
     return tuple(ngrams(doc.replace(' ', ''), 2))

Example #19

File: predicates.py Project: datamade/dedupe

def oneGramFingerprint(field):
    return (u''.join(sorted(set(ngrams(field.replace(' ', ''), 1)))).strip(),)

Example #20

File: predicates.py Project: datamade/dedupe

def twoGramFingerprint(field):
    if len(field) > 1:
        return (u''.join(sorted(gram.strip() for gram
                                in set(ngrams(field.replace(' ', ''), 2)))),)
    else:
        return ()

Example #21

File: predicates.py Project: datamade/dedupe

def commonFourGram(field):
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))

Example #22

File: predicates.py Project: shahin/dedupe

def commonSixGram(field):
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))

Example #23

def oneGramFingerprint(field):
    return (u''.join(sorted(ngrams(field, 1))).strip(), )