示例#1
0
def twoGramFingerprint(field):
    if len(field) > 1:
        return (u''.join(
            sorted(gram.strip()
                   for gram in set(ngrams(field.replace(' ', ''), 2)))), )
    else:
        return ()
示例#2
0
def commonSixGram(field):
    """Split the field into overlapping windows of 6 characters (spaces removed).

    Examples:
    .. code:: python
        > print(commonFourGram('John Woodward'))
        > {'JohnWo', 'Woodwa', 'hnWood', 'nWoodw', 'odward', 'ohnWoo', 'oodwar'}
    """
    return set(ngrams(field.replace(' ', ''), 6))
示例#3
0
def commonSixGram(field):
    """return 6-grams"""
    return ngrams(field, 6)
示例#4
0
def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)
示例#5
0
def commonSixGram(field):
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))
示例#6
0
def twoGramFingerprint(field):
    return (u''.join(sorted(gram.strip() for gram in ngrams(field, 2))), )
示例#7
0
def commonFourGram(field):
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))
示例#8
0
def oneGramFingerprint(field):
    return (u''.join(sorted(set(ngrams(field.replace(' ', ''), 1)))).strip(), )
示例#9
0
def commonSixGram(field: str) -> Set[str]:
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))
示例#10
0
文件： predicates.py 项目： 01-/dedupe 
 def preprocess(self, doc) :
     return tuple(ngrams(doc.replace(' ', ''), 2))
示例#11
0
def commonSixGram(field):
    """return 6-grams"""
    return ngrams(field, 6)
示例#12
0
def commonFourGram(field):
    """return 4-grams"""
    return ngrams(field, 4)
示例#13
0
def twoGramFingerprint(field) :
    return (u''.join(sorted(gram.strip() for gram in ngrams(field, 2))),)
示例#14
0
def oneGramFingerprint(field) :
    return (u''.join(sorted(ngrams(field, 1))).strip(),)
示例#15
0
 def preprocess(self, doc):
     return tuple(sorted(ngrams(" ".join(strip_punc(doc).split()), 2)))
示例#16
0
def commonFourGram(field: str) -> Set[str]:
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))
示例#17
0
 def preprocess(self, doc):
     return tuple(sorted(ngrams(" ".join(strip_punc(doc).split()), 2)))
示例#18
0
 def preprocess(self, doc):
     return tuple(ngrams(doc.replace(' ', ''), 2))
示例#19
0
def oneGramFingerprint(field):
    return (u''.join(sorted(set(ngrams(field.replace(' ', ''), 1)))).strip(),)
示例#20
0
def twoGramFingerprint(field):
    if len(field) > 1:
        return (u''.join(sorted(gram.strip() for gram
                                in set(ngrams(field.replace(' ', ''), 2)))),)
    else:
        return ()
示例#21
0
def commonFourGram(field):
    """return 4-grams"""
    return set(ngrams(field.replace(' ', ''), 4))
示例#22
0
def commonSixGram(field):
    """return 6-grams"""
    return set(ngrams(field.replace(' ', ''), 6))
示例#23
0
def oneGramFingerprint(field):
    return (u''.join(sorted(ngrams(field, 1))).strip(), )