def main(senna_path="",
sent="",
dep_model="",
batch=False,
stp_dir="",
init=False):
annotator = Annotator(senna_path, stp_dir, dep_model)
if not sent and batch:
sent = [
"He killed the man with a knife and murdered"
"him with a dagger.",
"He is a good boy.",
"He created the robot and broke it after making it.",
]
elif not sent:
sent = "He created the robot and broke it after making it."
if not batch:
click.echo("\n", sent, "\n")
sent = sent.split()
args = "-srl -pos".strip().split()
click.echo("conll:\n", annotator.get_conll_format(sent, args))
temp = annotator.get_annoations(sent, dep_parse=True)
click.echo("dep_parse:\n", temp["dep_parse"])
click.echo("chunk:\n", temp["chunk"])
click.echo("pos:\n", temp["pos"])
click.echo("ner:\n", temp["ner"])
click.echo("srl:\n", temp["srl"])
click.echo("syntaxTree:\n", temp["syntax_tree"])
click.echo("words:\n", temp["words"])
click.echo("skip gram\n", list(skipgrams(sent, n=3, k=2)))
else:
click.echo("\n\nrunning batch process", "\n", "=" * 20, "\n", sent,
"\n")
args = "-srl -pos".strip().split()
click.echo("conll:\n", annotator.get_conll_format(sent, args))
click.echo(Fore.BLUE + "CoNLL format is recommented for batch process")
class SemanticRoleLabeler:
def __init__(self, senna_path, stanford_parser_path):
self.senna_path = os.path.abspath(senna_path)
self.stanford_parser_path = os.path.abspath(stanford_parser_path)
self.annotator = Annotator(senna_dir=self.senna_path,
stp_dir=self.stanford_parser_path)
def get_srl(self, sentence):
if isinstance(sentence, str):
sentence = sentence.split()
return self.annotator.get_annoations(sentence)['srl']
class TestAnnotator:
def setup(self):
self.sent = "PgAdmin is the leading Open Source management tool for Postgres, the world’s most advanced Open Source database."
args = {"senna_dir": self.get_senna_path("pntl", "senna"), "save_all": True}
self.annotator = Annotator(**args)
self.process = self.annotator.get_annoations(self.sent, dep_parse=True)
print(self.process["chumk"])
def get_senna_path(self, *value):
temp = getcwd()
return join(temp, *value)
def main(senna_path='',
sent='',
dep_model='',
batch=False,
stp_dir='',
init=False):
annotator = Annotator(senna_path, stp_dir, dep_model)
if not sent and batch:
sent = [
"He killed the man with a knife and murdered"
"him with a dagger.", "He is a good boy.",
"He created the robot and broke it after making it."
]
elif not sent:
sent = "He created the robot and broke it after making it."
if not batch:
print("\n", sent, "\n")
sent = sent.split()
args = '-srl -pos'.strip().split()
print("conll:\n", annotator.get_conll_format(sent, args))
temp = annotator.get_annoations(sent, dep_parse=True)['dep_parse']
print('dep_parse:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['chunk']
print('chunk:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['pos']
print('pos:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['ner']
print('ner:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['srl']
print('srl:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['syntax_tree']
print('syntaxTree:\n', temp)
temp = annotator.get_annoations(sent, dep_parse=True)['words']
print('words:\n', temp)
print('skip gram\n', list(skipgrams(sent, n=3, k=2)))
else:
print("\n\nrunning batch process", "\n", "=" * 20, "\n", sent, "\n")
args = '-srl -pos'.strip().split()
print("conll:\n", annotator.get_conll_format(sent, args))
print(Fore.BLUE + "CoNLL format is recommented for batch process")
class ShallowParser:
def __init__(self):
print("Shallow Parser Initializing")
self.annotator = Annotator(senna_dir="./senna/")
self.stop_words = [
"a", "as", "able", "about", "above", "according", "accordingly",
"across", "actually", "after", "afterwards", "again", "against",
"aint", "all", "allow", "allows", "almost", "alone", "along",
"already", "also", "although", "always", "am", "among", "amongst",
"an", "and", "another", "any", "anybody", "anyhow", "anyone",
"anything", "anyway", "anyways", "anywhere", "apart", "appear",
"appreciate", "appropriate", "are", "arent", "around", "as",
"aside", "ask", "asking", "associated", "at", "available", "away",
"awfully", "be", "became", "because", "become", "becomes",
"becoming", "been", "before", "beforehand", "behind", "being",
"believe", "below", "beside", "besides", "best", "better",
"between", "beyond", "both", "brief", "but", "by", "cmon", "cs",
"came", "can", "cant", "cannot", "cant", "cause", "causes",
"certain", "certainly", "changes", "clearly", "co", "com", "come",
"comes", "concerning", "consequently", "consider", "considering",
"contain", "containing", "contains", "corresponding", "could",
"couldnt", "course", "currently", "definitely", "described",
"despite", "did", "didnt", "different", "do", "does", "doesnt",
"doing", "dont", "done", "down", "downwards", "during", "each",
"edu", "eg", "eight", "either", "else", "elsewhere", "enough",
"entirely", "especially", "et", "etc", "even", "ever", "every",
"everybody", "everyone", "everything", "everywhere", "ex",
"exactly", "example", "except", "far", "few", "ff", "fifth",
"first", "five", "followed", "following", "follows", "for",
"former", "formerly", "forth", "four", "from", "further",
"furthermore", "get", "gets", "getting", "given", "gives", "go",
"goes", "going", "gone", "got", "gotten", "greetings", "had",
"hadnt", "happens", "hardly", "has", "hasnt", "have", "havent",
"having", "he", "hes", "hello", "help", "hence", "her", "here",
"heres", "hereafter", "hereby", "herein", "hereupon", "hers",
"herself", "hi", "him", "himself", "his", "hither", "hopefully",
"how", "howbeit", "however", "i", "id", "ill", "im", "ive", "ie",
"if", "ignored", "immediate", "in", "inasmuch", "inc", "indeed",
"indicate", "indicated", "indicates", "inner", "insofar",
"instead", "into", "inward", "is", "isnt", "it", "itd", "itll",
"its", "its", "itself", "just", "keep", "keeps", "kept", "know",
"knows", "known", "last", "lately", "later", "latter", "latterly",
"least", "less", "lest", "let", "lets", "like", "liked", "list",
"likely", "little", "look", "looking", "looks", "ltd", "mainly",
"many", "may", "maybe", "me", "mean", "meanwhile", "merely",
"might", "more", "moreover", "most", "mostly", "much", "must",
"my", "myself", "name", "namely", "nd", "near", "nearly",
"necessary", "need", "needs", "neither", "never", "nevertheless",
"new", "next", "nine", "no", "nobody", "non", "none", "noone",
"nor", "normally", "not", "nothing", "novel", "now", "nowhere",
"obviously", "of", "off", "often", "oh", "ok", "okay", "old", "on",
"once", "one", "ones", "only", "onto", "or", "other", "others",
"otherwise", "ought", "our", "ours", "ourselves", "out", "outside",
"over", "overall", "own", "particular", "particularly", "per",
"perhaps", "placed", "please", "plus", "possible", "presumably",
"probably", "provides", "que", "quite", "qv", "rather", "rd", "re",
"really", "reasonably", "regarding", "regardless", "regards",
"relatively", "respectively", "right", "said", "same", "saw",
"say", "saying", "says", "second", "secondly", "see", "seeing",
"seem", "seemed", "seeming", "seems", "seen", "self", "selves",
"sensible", "sent", "serious", "seriously", "seven", "several",
"shall", "she", "should", "shouldnt", "since", "six", "so", "some",
"somebody", "somehow", "someone", "something", "sometime",
"sometimes", "somewhat", "somewhere", "soon", "sorry", "specified",
"specify", "specifying", "still", "sub", "such", "sup", "sure",
"ts", "take", "taken", "tell", "tends", "th", "than", "thank",
"thanks", "thanx", "that", "thats", "thats", "the", "their",
"theirs", "them", "themselves", "then", "thence", "there",
"theres", "thereafter", "thereby", "therefore", "therein",
"theres", "thereupon", "these", "they", "theyd", "theyll",
"theyre", "theyve", "think", "third", "this", "thorough",
"thoroughly", "those", "though", "three", "through", "throughout",
"thru", "thus", "to", "together", "too", "took", "toward",
"towards", "tried", "tries", "truly", "try", "trying", "twice",
"two", "un", "under", "unfortunately", "unless", "unlikely",
"until", "unto", "up", "upon", "us", "use", "used", "useful",
"uses", "using", "usually", "value", "various", "very", "via",
"viz", "vs", "want", "wants", "was", "wasnt", "way", "we", "wed",
"well", "were", "weve", "welcome", "well", "went", "were",
"werent", "what", "whats", "whatever", "when", "whence",
"whenever", "where", "wheres", "whereafter", "whereas", "whereby",
"wherein", "whereupon", "wherever", "whether", "which", "while",
"whither", "who", "whos", "whoever", "whole", "whom", "whose",
"why", "will", "willing", "wish", "with", "within", "without",
"wont", "wonder", "would", "would", "wouldnt", "yes", "yet", "you",
"youd", "youll", "youre", "youve", "your", "yours", "yourself",
"yourselves", "zero", "whose", "which", "is", ", ", "\\\\", "?",
"\\"
]
print("Shallow Parser Initialized")
def shallowParse(self, text):
if '?' not in text:
text = text + '?'
filterednpchunks = []
result = self.annotator.get_annoations([text])['chunk']
print(result)
chunkswithpositions = [] #For calculating surface indices
searchfrom = 0
for chunkpair in result:
position = text.find(chunkpair[0], searchfrom)
searchfrom = position + 1
length = len(chunkpair[0])
chunkswithpositions.append(
(chunkpair[0], chunkpair[1], position, length))
phrases = []
_phrase = []
for chunk in chunkswithpositions:
if chunk[1] == 'S-NP':
phrases.append([chunk])
continue
if chunk[1] == 'B-NP' or chunk[1] == 'I-NP':
_phrase.append(chunk)
continue
if chunk[1] == 'E-NP':
_phrase.append(chunk)
phrases.append(_phrase)
_phrase = []
for chunk in chunkswithpositions:
if chunk[1] == 'S-VP':
phrases.append([chunk])
continue
if chunk[1] == 'B-VP' or chunk[1] == 'I-VP':
_phrase.append(chunk)
continue
if chunk[1] == 'E-VP':
_phrase.append(chunk)
phrases.append(_phrase)
_phrase = []
print(phrases)
for phrase in phrases:
filteredchunk = []
filteredchunkstring = []
for word in phrase:
if word[0].lower() not in self.stop_words:
filteredchunk.append(word)
if len(filteredchunk) > 0:
filteredchunkstring = filteredchunk
filterednpchunks.append(filteredchunkstring)
print(filterednpchunks)
return filterednpchunks