def operator_precedence_features(): features = [] pass_num = 0 with open('exp_data_turk_new.json', "r") as f: exps = json.load(f) for exp_num in tqdm(range(len(exps))): exp = exps[exp_num] if exp_num == int(len(exps) / 4) or exp_num == int( len(exps) / 2) or exp_num == int(len(exps) / 4 * 3): print(len(features)) try: e = exp['exp'] new_sent = pre_process_sent(e) sent_tokenized = print_tokenized(new_sent) quote_words = new_predicate(sent_tokenized) raw_lexicon = add_new_predicate(quote_words) lex = lexicon.fromstring(raw_lexicon, True) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) for sent in sent_tokenized: for i, parse in enumerate(list(parser.parse(sent))): sem = parse_sem(str(parse.label()[0].semantics())) if sem != False: collect_features(sem, features) except: pass_num += 1 counter = Counter(features) prt = list(list(zip(*list(counter.most_common())))[0]) print(prt) print(len(prt)) print(pass_num, len(exps))
def speech_reco_core(): with open('/home/crazykoe/turtlebotws/lexicon.txt', 'r') as file: myLexicon = file.read() lex = lexicon.fromstring(myLexicon, True) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) r = sr.Recognizer() with sr.Microphone() as source: print("What do you need?") audio = r.listen(source) try: print("I think you said " + r.recognize_google(audio) + ". Got it!") except sr.UnknownValueError: print("Please say it again.") except sr.RequestError as e: print("The service is down".format(e)) requestuni = r.recognize_google(audio) request = requestuni.encode("utf-8") cmd = request parses = list(parser.parse(cmd.lower().split())) if len(parses) != 0: (token, op) = parses[0][()].label() if token.semantics() is not None: output = str(token.semantics()) match = re.findall( "(?:action\((\w+)\) & target\((\w+)(?:\((\w+)\))?\)(?: &)?)+", output) if len(match) == 1: robotmove = array_msg() robotmove.action = match[0][0] robotmove.target = match[0][1] robotmove.name = match[0][2] robotmove.cmdaction = '' robotmove.targetroom = '' robotmove.names = '' else: robotmove = array_msg() robotmove.action = match[0][0] robotmove.target = match[0][1] robotmove.name = match[0][2] robotmove.cmdaction = match[1][0] robotmove.targetroom = match[1][1] robotmove.names = match[1][2] else: print('Unable to parse') return (robotmove)
def rule_features(): lis = [] pass_num = 0 with open('exp_data_new.json', "r") as f: exps = json.load(f) for exp_num in tqdm(range(len(exps))): exp = exps[exp_num] try: e = exp['exp'] new_sent = pre_process_sent(e) sent_tokenized = print_tokenized(new_sent) raw_lexicon = add_new_predicate(sent_tokenized) lex = lexicon.fromstring(raw_lexicon, True) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) for parse in parser.parse(sent_tokenized[0].split()): recurse_print(parse, lis) except: pass_num += 1 pass with open('combrules.json', "w") as f: json.dump(lis, f)
# Construct the lexicon lex = fromstring(""" :- S, NP, N, VP # Primitive categories, S is the target primitive Det :: NP/N # Family of words Pro :: NP TV :: VP/NP Modal :: (S\\NP)/VP # Backslashes need to be escaped I => Pro # Word -> Category mapping you => Pro the => Det # Variables have the special keyword 'var' # '.' prevents permutation # ',' prevents composition and => var\\.,var/.,var which => (N\\N)/(S/NP) will => Modal # Categories can be either explicit, or families. might => Modal cook => TV eat => TV mushrooms => N parsnips => N bacon => N """)
from nltk.ccg import chart, lexicon from nltk.ccg.chart import printCCGDerivation print('''============================== === Lexicon l3 === ============================== ''') l3 = lexicon.fromstring( ''' :- S, NP Justin => NP {\P.P(j)} Keisha => NP {\P.P(k)} somebody => NP {\P.exists x.(person(x) & P(x))} everybody => NP {\P.forall x.(person(x) -> P(x))} admires => (S\\NP)/NP {\Y.(\Z.Z(\z.Y(\y.admire(z,y))))} complains => S\\NP {complain} ''', True) print(l3) print() print( '''==================================================================================== === Derivation for \'somebody admires everybody\' obtained with ApplicationRuleSet === === The semantics is the expected one. === ====================================================================================''' ) parser1 = chart.CCGChartParser(l3, chart.ApplicationRuleSet) parses = list(parser1.parse("somebody admires everybody".split())) printCCGDerivation(parses[0])
# Construct the lexicon lex = fromstring(''' :- S, NP, N, VP # Primitive categories, S is the target primitive Det :: NP/N # Family of words Pro :: NP TV :: VP/NP Modal :: (S\\NP)/VP # Backslashes need to be escaped I => Pro # Word -> Category mapping you => Pro the => Det # Variables have the special keyword 'var' # '.' prevents permutation # ',' prevents composition and => var\\.,var/.,var which => (N\\N)/(S/NP) will => Modal # Categories can be either explicit, or families. might => Modal cook => TV eat => TV mushrooms => N parsnips => N bacon => N ''')
def parse_tokens(one_sent_tokenize, raw_lexicon): """ CYK algorithm for parsing a tokenized sentence into a parse tree. We implement our own, as solely using NLTK's CCGChartParser and the grammar we came up won't allow for the parses we desired. As we are not linguists, we found it easier to change the code than figure out possible problems with our grammar. Outputs the last row of the CYK datastructure as possible parses for the sentence * Each element in the row is string version of nltk.tree.Tree (sort of, we actually construct our own tree based on the tree provided by NLTK) Arguments: one_sent_tokenize (arr) : array of string tokens representing a sentence raw_lexicon (str) : string representation of lexicon (grammar and vocabulary rep of a language) Returns: (arr) : list of possible parses, read comment above for more """ try: beam_lexicon = copy.deepcopy(raw_lexicon) CYK_form = [[[token] for token in one_sent_tokenize]] CYK_sem = [[]] for layer in range(1,len(one_sent_tokenize)): layer_form = [] layer_sem = [] lex = lexicon.fromstring(beam_lexicon, True) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) for col in range(0,len(one_sent_tokenize)-layer): form = [] sem_temp = [] word_index = 0 st = col+0 ed = st+layer for splt in range(st,ed): words_L = CYK_form[splt-st][st] words_R = CYK_form[ed-splt-1][splt+1] for word_0 in words_L: for word_1 in words_R: try: for parse in parser.parse([word_0, word_1]): (token, op) = parse.label() categ = token.categ() sem = token.semantics() word_name = '$Layer{}_Horizon{}_{}'.format(str(layer), str(col),str(word_index)) word_index+=1 entry = "\n\t\t"+word_name+' => '+str(categ)+" {"+str(sem)+"}" if str(sem)+'_'+str(categ) not in sem_temp: form.append((parse,word_name,entry,str(sem))) sem_temp.append(str(sem)+'_'+str(categ)) except: pass add_form = [] for elem in form: parse, word_name, entry,sem_ = elem add_form.append(word_name) beam_lexicon = beam_lexicon+entry layer_sem.append(sem_) layer_form.append(add_form) CYK_form.append(layer_form) CYK_sem.append(layer_sem) return CYK_sem[-1] except: return []
from nltk.ccg import chart, lexicon lex = lexicon.fromstring(''' :- S, N, NP Steel => N is => (S\\N)/NP an => NP/N alloy => N contains => (S\\N)/N carbon => N Does => NP/NP Ferrite => NP have => (S\\NP)/N high => N/N hardness => N Which => S/(S\\NP)/N material => N has => (S\\NP)/NP the => NP/N lowest => N/N tensile => N/N strength => N ''', False) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) for parse in parser.parse("Steel is an alloy".split()): chart.printCCGDerivation(parse) break for parse in parser.parse("Steel contains carbon".split()):
lex2 = lexicon.fromstring(r""" :- NN, INP, ADJ, DET, IN DET :: NN/NN ADJ :: NN/NN IN :: (NN\NN)/NN ADV :: NN/ADJ QS :: NN/NN same => ADJ {\x.same_(x)} same => IN {\x y.same_(x,y)} material => NN {'material'} color => NN {'color'} shape => NN {'shape'} size => NN {'size'} as => IN {\x y.as(x,y)} of => IN {\x y.of(x,y)} to => DET {\x.relate(x)} the => DET {\P.(P)} a => DET {\P.(P)} an => DET {\P.(P)} any => DET {\P.(P)} metallic => ADJ {\x.filter_material(x,'metal')} metal => ADJ {\x.filter_material(x,'metal')} shiny => ADJ {\x.filter_material(x,'metal')} rubber => ADJ {\x.filter_material(x,'rubber')} matte => ADJ {\x.filter_material(x,'rubber')} gray => ADJ {\x.filter_color(x,'gray')} red => ADJ {\x.filter_color(x,'red')} blue => ADJ {\x.filter_color(x,'blue')} green => ADJ {\x.filter_color(x,'green')} brown => ADJ {\x.filter_color(x,'brown')} purple => ADJ {\x.filter_color(x,'purple')} cyan => ADJ {\x.filter_color(x,'cyan')} yellow => ADJ {\x.filter_color(x,'yellow')} big => ADJ {\x.filter_size(x,'large')} large => ADJ {\x.filter_size(x,'large')} small => ADJ {\x.filter_size(x,'small')} tiny => ADJ {\x.filter_size(x,'small')} left => ADJ {\x.left(x,'prueba')} left => NN {'left'} cube => NN {'cube'} block => NN {'cube'} sphere => ADJ {\x.filter_shape(x,'sphere')} spheres => NN {'sphere'} ball => NN {'sphere'} cylinder => NN {'cylinder'} what_is => QS {\x.query_(x)} are_there => QS {\x.query_(x)} is_there => QS {\x.query_(x)} object => NN {scene} thing => NN {scene} it => NN {scene} """, include_semantics=semantics)
""" from nltk.ccg import chart, lexicon question = "Are there any other things that are the same shape as the big metallic object?" parse = "exist(same_shape(unique(filter_material(filter_size(scene, 'large'), 'metal'))))" lex = lexicon.fromstring(r""" :- NN, INP, ADJ, DET, IN DET :: NN/NN ADJ :: NN/NN IN :: (NN\NN[comp])/NN same => ADJ {\x.same(x)} shape => NN {'shape'} as => IN {\x y.pair(x,y)} the => DET {\P.unique(P)} big => ADJ {\x.filter_size(x,'large')} metallic => ADJ {\x.filter_material(x,'metal')} object => NN {scene}""", include_semantics=True) parser = chart.CCGChartParser(lex, chart.DefaultRuleSet) results = list( parser.parse("the same shape as the big metallic object".split())) for parse in results[:3]: chart.printCCGDerivation(parse)
from nltk.ccg import chart, lexicon import numpy as np semantics = True lex = lexicon.fromstring(r""" :- NN, DET, ADJ DET :: NN/NN ADJ :: NN/NN the => DET {\x.unique(x)} blue => ADJ {\x.color(x,red)} blue => ADJ {\x.color(x,blue)} red => ADJ {\x.color(x,red)} red => ADJ {\x.color(x,blue)} ball => NN {ball} """, include_semantics=semantics) data = [ ("the blue ball".split(), "unique(color(ball,blue))"), ("the red ball".split(), "unique(color(ball,red))"), ]