def pos_tag_tokens(line, lang):
"""
Do POS-tagging but return tuples for each input word
:type line: list(str) An already tokenized line
:type lang: str The language (2-letter code)
"""
iso3 = ('sve' if lang[:2] == 'sv' else 'eng')
if iso3 in ht_cache:
ht = ht_cache[iso3]
else:
if iso3 == 'eng':
model = 'en_wsj.model'
enc = 'utf-8'
else: # Swedish
model = 'suc-suctags.model'
enc = 'ISO-8859-1'
# build the tagger
if platform.system() == 'Windows':
ht = HunposTagger(
model,
path_to_bin=r'.\thirdparty\hunpos-win\hunpos-tag.exe',
encoding=enc)
else:
ht = HunposTagger(model, path_to_bin='./hunpos-tag', encoding=enc)
# cache it
ht_cache[iso3] = ht
tuples = ht.tag(line)
return tuples
def __init__(self,
tok=False,
wtok=False,
stok=False,
pos=False,
stem=False,
pos_model=None,
abbrev_set=None,
stok_model=None):
"""@param abbrev_set: a set of frequent abbreviations."""
if tok:
wtok = True
stok = True
if wtok:
self.wordTokenizer = PunktWordTokenizer()
#self.punktSplitter = re.compile(r"^([\w\d]+)([.?,:;!])$", re.UNICODE)
self.punktSplitter = re.compile(r"^(.+)([.?,:;!])$", re.UNICODE)
# Bragantino,2006.In fix this shit
if stok:
if stok_model is not None:
try:
self.senTokenizer = stok_model
except LookupError:
sys.stderr.write("WARNING: tokenizer cannot be loaded")
sys.stderr.write(
"WARNING: using an untrained sen_tokenizer")
self.senTokenizer = PunktSentenceTokenizer()
else:
try:
self.senTokenizer = nltk.data.load(
'tokenizers/punkt/english.pickle')
except LookupError:
sys.stderr.write(
"WARNING: english tokenizer cannot be loaded, nltk.data does not contain in!"
)
sys.stderr.write(
"WARNING: using an untrained sen_tokenizer")
self.senTokenizer = PunktSentenceTokenizer()
self.abbrev_set = (set(abbrev_set)
if abbrev_set is not None else set())
if pos:
if pos_model is not None:
self.posTagger = HunposTagger(pos_model, encoding="utf-8")
else:
self.posTagger = HunposTagger(os.path.join(
os.environ['HUNPOS'], 'english.model'),
encoding="utf-8")
if stem:
self.stemmer = WordNetLemmatizer()
def filter_on_basis_pos_tag(self):
hunpos_tagger = HunposTagger(HunPosModelPath, HunPosTagPath)
filtered_list = list()
def check_np(np):
print "Actual NP %s\n" % np
try:
#__list = hunpos_tagger.tag(nltk.wordpunct_tokenize(np.encode("utf-8")))
__list = hunpos_tagger.tag(nltk.wordpunct_tokenize(np))
print __list
except Exception as e:
print "This bullsht string is being ignored %s" % np
return None
if not set.intersection(set(["NNP", "NN", "NNS"]),
set([__tag for (token, __tag) in __list])):
return None
result = [
__token for (__token, __tag) in __list
if not __tag in ["RB", "CD", "FW"]
]
print "Stripped off NP %s \n" % " ".join(result)
return " ".join(result)
for __e in self.result:
filtered_list.append(check_np(__e))
return filter(None, filtered_list)
def custom_ner(self):
ner = list()
regexp_grammer = r"NER:{<IN><NN.*><NN.*>?}"
__parser = nltk.RegexpParser(regexp_grammer)
hunpos_tagger = HunposTagger(HunPosModelPath, HunPosTagPath)
for __sentence in self.sentences:
try:
tagged = hunpos_tagger.tag(
nltk.word_tokenize(__sentence.encode("utf-8")))
except Exception as e:
hunpos_tagger = HunposTagger(HunPosModelPath, HunPosTagPath)
tagged = hunpos_tagger.tag(
nltk.word_tokenize(__sentence.encode("utf-8")))
tree = __parser.parse(tagged)
for subtree in tree.subtrees(filter=lambda t: t.label() == 'NER'):
l = " ".join([
e[0] for e in subtree.leaves()
if e[1] == 'NNP' or e[1] == 'NNS' or e[1] == 'NN'
])
ner.append(l.lower())
result = sorted(Counter(ner).items(), reverse=True, key=lambda x: x[1])
return result
def filter_on_basis_pos_tag(self):
"""
pos tagging of noun phrases will be done, and if the noun phrases contains some adjectives or RB or FW,
it will be removed from the total noun_phrases list
Any Noun phrases when split, if present in self.list_to_exclude will not be included in the final result
for Example:
self.list_to_exclude = ["food", "i", "service", "cost", "ambience", "delhi", "Delhi", "place", "Place"]
noun_phrase = "great place"
"""
print "{0} These noun phrases will be removed from the noun phrases {1}".format(
bcolors.OKBLUE, bcolors.RESET)
print "{0} List To Exclude {1}".format(bcolors.OKBLUE, bcolors.RESET)
print self.list_to_exclude
print "\n"
print "{0} Common name entities {1}".format(bcolors.OKBLUE,
bcolors.RESET)
print self.common_ners
print "\n"
hunpos_tagger = HunposTagger(HunPosModelPath, HunPosTagPath)
filtered_list = list()
for __e in self.result:
__list = [
pos_tag for (np, pos_tag) in hunpos_tagger.tag(
nltk.wordpunct_tokenize(__e.get("name")))
]
if bool(
set.intersection(set(__e.get("name").split(" ")),
set(self.list_to_exclude))):
print __e.get("name")
pass
#elif __e.get("name") in self.common_ners:
# pass
elif "RB" == __list[0] or "CD" in __list or "FW" in __list:
pass
else:
filtered_list.append(__e)
return filtered_list
def work():
global conversion
pos_tagger = HunposTagger(path_to_model=config.path_list.hunpos_model,
path_to_bin=config.path_list.hunpos_bin)
conversion = inflect.engine()
with open(config.path_list.en_stopwords, 'r', encoding='utf-8') as f:
stopwords = set(f.read().split('\n'))
with open(config.path_list.en_kp_list, 'r', encoding='utf-8') as f:
lists = f.read().split('\n')
print('load kp_list done.')
with open(config.path_list.input, 'r', encoding='utf-8') as f:
data = []
for line in f.read().split('\n'):
if line == '':
continue
data.append(line.lower())
res1 = segment(data, pos_tagger)
res2 = pattern_filter(lists, res1, stopwords)
json_gen(res2)
print('prework done.')
pos_tagger.close()
def filter_on_basis_pos_tag(self):
"""
pos tagging of noun phrases will be d
one, and if the noun phrases contains some adjectives or RB or FW,
it will be removed from the total noun_phrases list
Any Noun phrases when split, if present in self.list_to_exclude will not be included in the final result
for Example:
self.list_to_exclude = ["food", "i", "service", "cost", "ambience", "delhi", "Delhi", "place", "Place"]
noun_phrase = "great place"
"""
hunpos_tagger = HunposTagger(HunPosModelPath, HunPosTagPath)
filtered_list = list()
for __e in self.result:
__list = [pos_tag for (np, pos_tag) in hunpos_tagger.tag(nltk.wordpunct_tokenize(__e.get("name").encode("ascii", "ignore")))]
if set.intersection(set(__list), set(["FW", "CD", "LS"])):
print "This will be droppped out of total noun phrases %s"%__e.get("name")
else:
filtered_list.append(__e)
return filtered_list
def get_hunpos_tagger():
hunmorph_dir = os.environ['HUNMORPH_DIR']
hunpos_binary = os.path.join(hunmorph_dir, 'hunpos-tag')
hunpos_model = os.path.join(hunmorph_dir, 'en_wsj.model')
return HunposTagger(hunpos_model, hunpos_binary)
__author__ = 'pengzhang'
# New Version
# chunking the tuples into three arguments
import copy
import json
import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.tag.hunpos import HunposTagger
###############
# load data
tuple_anno = json.load(open('../abstract/binary_extracted_train.json'))
###############
ht = HunposTagger('../en_wsj.model')
p_list = ['NN', 'NNS', 'NNP', 'NNPS', 'PRP', 'PRP$', 'UH']
s_list = ['NN', 'JJ', 'JJS', 'JJR', 'DT', 'NNS', 'NNP', 'NNPS', 'PRP', 'PRP$']
s_jj_list = ['JJ', 'JJS', 'JJR', 'DT']
verb_list = ['VB', 'VBG', 'VBD', 'VBZ', 'VBN', 'VBP']
obj_list = json.load(open('../abstract/abstract_obj_list.json'))
location_list = ['bed','bar','pad','lilypad','shelf','house','coatrack','endtable','left','right','center','top',\
'front','middle','back','ground','cartoon','monkeybar','petbed','rope','footstool','bat']
all_obj_list = []
for i in range(len(obj_list)):
all_obj_list += obj_list[i].values()[0]
all_obj_list = set(all_obj_list)
all_obj_list = list(all_obj_list)
def __init__(self, model_fn=None):
self.tokenizer = NOTokenizer()
self.tagger = HunposTagger(NNO_TAGGER_DEFAULT_MODEL_FN,
hunpos_tag_bin(),
encoding='utf-8')