Python tokenize Examples

Example #1

def is_good(dispute):
    keywords, claim, matchwords = dispute
    claimwords = set(tokenize(claim.lower()))
    claimkeywords = claimwords - okwords
    textwords = set([word.lower() for word in matchwords])
    claim_not_text = claimkeywords - textwords
    return len(claim_not_text) == 0

Example #2

File: allwords_filter.py Project: BenTrem/think-link

def is_good(dispute):
	keywords,claim,matchwords = dispute
	claimwords = set(tokenize(claim.lower()))
	claimkeywords = claimwords - okwords
	textwords = set([word.lower() for word in matchwords])
	claim_not_text = claimkeywords - textwords
	return len(claim_not_text) == 0

Example #3

File: features2.py Project: BenTrem/think-link

def add_shared_props(item):
	item['matchwords'] = get_trimmed_match(item)	
	item['trimmedmatch'] = " ".join(item['matchwords'])			
	item['claimwords'] = tokenize(item['claimtext'])

Example #4

def add_shared_props(item):
    item['matchwords'] = get_trimmed_match(item)
    item['trimmedmatch'] = " ".join(item['matchwords'])
    item['claimwords'] = tokenize(item['claimtext'])