def compile ( syms, clss , code , nwy=0 ): """ create cognitive or generative semantic procedure from code arguments: syms - symbol table clss - type of procedure: 'c'=cognitive, 'g'=generative code - list of semantic commands nwy - type of grammar rule returns: cognitive or generative procedure on success, None otherwise """ inps = ellyDefinitionReader.EllyDefinitionReader(code) if clss == 'c': cp = cognitiveProcedure.CognitiveProcedure(syms,inps,nwy) return cp if cp.logic != None else None elif clss == 'g': gp = generativeProcedure.GenerativeProcedure(syms,inps) return gp if gp.logic != None else None else: print ( 'bad semantic procedure class' , file=sys.stderr ) return None
def inpT(self, system, part): """ inherited method to compose table input arguments: self - system - which set of table definitions? part - which table to read in? returns: EllyDefinitionReader on success, None otherwise """ basn = ellyConfiguration.baseSource if not basn[-1] == '/': basn += '/' suf = '.' + part + '.elly' filn = basn + system + suf if not os.path.exists(filn): filn = basn + ellyConfiguration.defaultSystem + suf print('reading', filn, end=' ', file=sys.stderr) rdr = ellyDefinitionReader.EllyDefinitionReader(filn) if rdr.error != None: self.errors.append(rdr.error) print(': failed', file=sys.stderr) return None else: print(': succeeded', file=sys.stderr) return rdr
def _genp(self, ls): """ create a generative procedure arguments: self - ls - list of command strings returns: GenerativeProcedure """ din = ellyDefinitionReader.EllyDefinitionReader(ls) return generativeProcedure.GenerativeProcedure(self.context.syms, din)
def __init__(self, stb, gtb, ptb): """ create parse tree generator with grammar and syntax type patterns arguments: self - stb - symbol table gtb - grammar ptb - patterns """ # print "at ParseTreeBottomUp" inp = ellyDefinitionReader.EllyDefinitionReader(['obtain']) self.litg = generativeProcedure.GenerativeProcedure(stb, inp) inp = ellyDefinitionReader.EllyDefinitionReader([]) self.litc = cognitiveProcedure.CognitiveProcedure(stb, inp) self.ntyp = stb.getSyntaxTypeCount() self.gtb = gtb self.ptb = ptb self.ctx = None # set by ParseTree, if at all super(ParseTreeBottomUp, self).__init__() # print "back in ParseTreeBottomUp" self._pls = 0
def _cogp(self, ls): """ create a cognitive procedure arguments: self - ls - list of command strings returns: CognitiveProcedure """ din = ellyDefinitionReader.EllyDefinitionReader(ls) return cognitiveProcedure.CognitiveProcedure(self.context.syms, din)
else: print ( '>{0:3d} {1} {2}'.format(loc,com,arg) ) cod = cod[dl:] loc += dl return noe if __name__ == "__main__": import ellyDefinitionReader import symbolTable stbd = symbolTable.SymbolTable() print ( 'generative semantic compilation test' ) srcd = sys.argv[1] if len(sys.argv) > 1 else 'generativeDefinerTest.txt' inpd = ellyDefinitionReader.EllyDefinitionReader(srcd) if inpd.error != None: print ( "cannot read procedure definition" , file=sys.stderr ) print ( inpd.error , file=sys.stderr ) sys.exit(1) print ( 'input=' , srcd ) codg = compileDefinition(stbd,inpd) if codg == None: print ( "semantic compilation error" , file=sys.stderr ) sys.exit(1) print ( len(codg) , 'code elements in procedure' ) showCode(codg)
return '--' class Ctx(object): # dummy interpretive context """ dummy interpretive context class """ def __init__(self): """ dummy method """ self.tokns = [] self.wghtg = Wtg() name = sys.argv[1] if len(sys.argv) > 1 else 'test' deep = int(sys.argv[2]) if len(sys.argv) > 2 else 100 base = ellyConfiguration.baseSource + '/' rdr = ellyDefinitionReader.EllyDefinitionReader(base + name + '.g.elly') if rdr.error != None: print(rdr.error, file=sys.stderr) sys.exit(1) print('loading', '[' + base + name + '.g.elly]', len(rdr.buffer), 'lines') stbu = symbolTable.SymbolTable() gtbu = grammarTable.GrammarTable(stbu, rdr) ctxu = Ctx() tksu = ctxu.tokns tree = ParseTreeWithDisplay(stbu, gtbu, None, ctxu) print() print(tree) print() print(dir(tree))
# # unit test # if __name__ == '__main__': import sys import ellyToken import ellyConfiguration import ellyDefinitionReader import inflectionStemmerEN filn = sys.argv[1] if len(sys.argv) > 1 else 'default' basn = ellyConfiguration.baseSource + '/' dfn = ellyDefinitionReader.EllyDefinitionReader(basn + filn + '.stl.elly') if dfn.error != None: print >> sys.stderr, dfn.error sys.exit(1) print dfn.linecount(), 'definition lines for', filn + '.stl.elly' try: inf = inflectionStemmerEN.InflectionStemmerEN() suf = SuffixTreeLogic(dfn) except ellyException.TableFailure: print >> sys.stderr, 'cannot load stemming tables' sys.exit(1) suf.infl = inf # print 'suf=' , suf # print 'index=' , map(lambda x: ellyChar.toChar(ellyChar.toIndex(x)) , suf.indx.keys()) print ''
acs.append(semanticCommand.Copn[opn] + ' ' + s) print(', '.join(acs)) # # unit test # if __name__ == "__main__": import ellyDefinitionReader ustb = symbolTable.SymbolTable() usrc = sys.argv[1] if len(sys.argv) > 1 else 'cognitiveDefinerTest.txt' uinp = ellyDefinitionReader.EllyDefinitionReader(usrc) if uinp.error != None: print(uinp.error, file=sys.stderr) sys.exit(1) ucod = convertDefinition(ustb, uinp, 2) if ucod == None: print("conversion error", file=sys.stderr) sys.exit(1) print(len(ucod), "clauses from", usrc) showCode(ucod)
global _table # must declare in order to set! # print ( 'before ntb=' , _table ) _table = table # print ( 'after ntb=' , _table ) # # unit test # if __name__ == '__main__': import ellyDefinitionReader inp = 'test' if len(sys.argv) == 1 else sys.argv[1] pnm = inp + '.n.elly' rdr = ellyDefinitionReader.EllyDefinitionReader(pnm) if rdr.error != None: print ( rdr.error ) sys.exit(1) ntb = nameTable.NameTable(rdr) print ( 'table loaded=' , ntb.filled() ) ntb.dump() setUp(ntb) while True: # test examples from standard input sys.stdout.write('> ')
return ru # # unit test # if __name__ == '__main__': import ellyConfiguration import dumpEllyGrammar import punctuationRecognizer filn = sys.argv[1] if len(sys.argv) > 1 else 'test' sym = symbolTable.SymbolTable() # print sym base = ellyConfiguration.baseSource + '/' inp = ellyDefinitionReader.EllyDefinitionReader(base + filn + '.g.elly') if inp.error != None: print inp.error sys.exit(1) print 'reading', '[' + filn + ']', len( inp.buffer), 'lines of rule definitions' try: gtb = GrammarTable(sym, inp) pnc = punctuationRecognizer.PunctuationRecognizer(sym) # print gtb dumpEllyGrammar.dumpAll(sym, gtb, 5) except ellyException.TableFailure: print >> sys.stderr, 'exiting'
if __name__ == '__main__': import sys arg = sys.argv[1:] if len(arg) > 0 and arg[0].lower() == '-zh': ellyConfiguration.language = arg.pop(0)[1:].upper() import ellyDefinitionReader import stopExceptions base = ellyConfiguration.baseSource dfs = base + (arg[1] if len(arg) > 1 else 'default') + '.sx.elly' # print ( 'reading exceptions from:' , dfs ) inp = ellyDefinitionReader.EllyDefinitionReader(dfs) if inp.error != None: print('cannot read stop exceptions', file=sys.stderr) print(inp.error, file=sys.stderr) sys.exit(1) stpxs = stopExceptions.StopExceptions(inp) tst = arg[0] if len(arg) > 0 else 'sentenceTestData.txt' ins = open(tst, 'r') rdr = EllySentenceReader(ins, stpxs) # print () while True: sents = rdr.getNext() if sents == None or len(sents) == 0: break
return True # successful match # # unit test # if __name__ == "__main__": import sys import ellyDefinitionReader import ellyException basn = ellyConfiguration.baseSource + '/' filn = sys.argv[1] if len(sys.argv) > 1 else 'test' mrdr = ellyDefinitionReader.EllyDefinitionReader(basn + filn + '.m.elly') print("loading rules for", filn) if mrdr.error != None: print(mrdr.error, file=sys.stderr) sys.exit(1) mtbl = macroTable.MacroTable(mrdr) sbuf = SubstitutionBuffer(mtbl) print('enter text for macro substitution') while True: try: sys.stdout.write("> ") sys.stdout.flush() l = sys.stdin.readline()
ix = 0 for gv in globs: # initialize global variables vr = 'gp' + str(ix) # variable names = gp0, gp1, ... eb.setGlobalVariable(vr, gv) print vr, '=', gv ix += 1 if dl >= 0: # check for specific depth limit eb.ptr.setDepth(dl) # if so, set it base = ellyConfiguration.baseSource sent = base + system + '.sx.elly' # get stop punctuation exceptions if not os.path.isfile(sent): sent = base + ellyConfiguration.defaultSystem + '.sx.elly' ind = ellyDefinitionReader.EllyDefinitionReader(sent) if ind.error != None: print >> sys.stderr, 'cannot read stop exceptions' print >> sys.stderr, ind.error sys.exit(1) try: exs = stopExceptions.StopExceptions(ind) rdr = ellySentenceReader.EllySentenceReader(sys.stdin, exs) # set up sentence reader except ellyException.TableFailure as e: print >> sys.stderr, '** input initialization failure'
# ----------------------------------------------------------------------------- """ vocabulary element from external table lookup """ import sys import ellyBits import ellyException import ellyDefinitionReader import generativeProcedure import unicodedata gens = ['*'] # define generative procedure for translation geno = ['obtain'] # default generative procedure inptx = ellyDefinitionReader.EllyDefinitionReader(geno) obtnp = generativeProcedure.GenerativeProcedure(None, inptx) class VocabularyElement(object): """ vocabulary term plus definition attributes: chs - list of chars in term cat - syntactic category syf - syntactic flags smf - semantic flags bia - initial plausibility score con - associated concept gen - generative procedure
import dumpEllyGrammar import parseTest import stat import os mode = os.fstat(0).st_mode # to check for redirection of stdin (=0) interact = not (stat.S_ISFIFO(mode) or stat.S_ISREG(mode)) ctx = parseTest.Context() # dummy interpretive context for testing tre = parseTest.Tree(ctx.syms) # dummy parse tree for testing print '' basn = ellyConfiguration.baseSource + '/' filn = sys.argv[1] if len( sys.argv) > 1 else 'test' # which FSA definition to use inp = ellyDefinitionReader.EllyDefinitionReader(basn + filn + '.p.elly') print 'pattern test with', '<' + filn + '>' if inp.error != None: print inp.error sys.exit(1) patn = None try: patn = PatternTable(ctx.syms, inp) # try to define FSA except ellyException.TableFailure: print 'no pattern table generated' sys.exit(1) print len(patn.indx), 'distinct FSA state(s)'
def __init__(self, dta): """ initialization of vocabulary object from retrieved record arguments: self - dta - what DB support returns throws: FormatFailure on error """ self._ln = 0 rec = dta[1] # data record found for search key # print ( 'voc rec=' , rec , file=sys.stderr ) r = rec.split('=:') # split off term in data record if len(r) <= 1: return # the '=:' is mandatory d = r[1].strip().split(' ') # definition is right of '=:' # print ( 'VEntry: define as' , d , file=sys.stderr ) if len(d) < 4: return # it should have at least 4 parts ur = r[0].strip() # term left of '=:' self.chs = list(ur) # save it self.cat = int(d.pop(0)) # syntactic category # print ( ' full term=' , ''.join(self.chs) , file=sys.stderr ) sy = d.pop(0) nb = len(sy) * 4 self.syf = ellyBits.EllyBits(nb) # allocate bits self.syf.reinit(sy) # set syntactic features sm = d.pop(0) nb = len(sm) * 4 self.smf = ellyBits.EllyBits(nb) # allocate bits self.smf.reinit(sm) # set semantic features self.bia = int(d.pop(0)) # save initial plausibility if len(d) > 0: # any concept? self.con = d.pop(0).upper() # if so, save it else: self.con = '-' # print ( ' translation=' , d , file=sys.stderr ) if len(d) == 0: # no further definition? self.gen = obtnp # if so, then use default procedure self._nt = 0 # i.e. no translation elif d[0][0] == '=': # simple translation? dfs = ' '.join(d) # just in case translation had spaces # print ( 'def ext=' , dfs , file=sys.stderr ) pls = ['append ' + dfs[1:]] inpts = ellyDefinitionReader.EllyDefinitionReader(pls) self.gen = generativeProcedure.GenerativeProcedure(None, inpts) self._nt = 1 elif d[0][0] == '(': # get predefined procedure inpts = ellyDefinitionReader.EllyDefinitionReader([d[0]]) self.gen = generativeProcedure.GenerativeProcedure(None, inpts) self._nt = 0 else: # otherwise, set for selection of translation # print ( 'multi selection, d=' , d , file=sys.stderr ) cm = 'pick LANG (' # construct instruction to select for p in d: if p[-1] == ',': p = p[:-1] cm += p + '#' # build selection clauses cm += ')' gens[0] = cm # replace action # print ( 'cm=' , cm ) # print ( 'gens=' , gens ) inpts = ellyDefinitionReader.EllyDefinitionReader(gens) self.gen = generativeProcedure.GenerativeProcedure(None, inpts) if self.gen == None: print('vocabulary generative semantic failure', file=sys.stderr) print('gens=', gens, file=sys.stderr) raise ellyException.FormatFailure # print ( 'vocabulary gen.logic' ) # generativeDefiner.showCode(self.gen.logic) self._nt = len(d) self._ln = len(self.chs)
import ellyDefinitionReader import procedureTestFrame from generativeDefiner import showCode frame = procedureTestFrame.ProcedureTestFrame() phr = frame.phrase ctx = frame.context stb = ctx.syms args = sys.argv[1:] if len(sys.argv) > 1 else [ 'testProcedure.0.txt' ] for srcn in args: # test all specified input files ctx.clearLocalStack() # reset context ctx.clearBuffers() print "------------" , srcn inp = ellyDefinitionReader.EllyDefinitionReader(srcn) if inp.error != None: # check if input readable print >> sys.stderr, "cannot read procedure definition" , srcn print >> sys.stderr, inp.error continue for ln in inp.buffer: # echo input file print ln print '' gp = GenerativeProcedure(stb,inp) # compile procedure print '*CODE*' if gp.logic != None: showCode(gp.logic) # dump procedure logic else: print 'no logic'
import ellyConfiguration import ellyDefinitionReader import parseTest import stat import os mode = os.fstat(0).st_mode # to check for redirection of stdin (=0) interact = not ( stat.S_ISFIFO(mode) or stat.S_ISREG(mode) ) ctx = parseTest.Context() # dummy interpretive context for testing tre = parseTest.Tree(ctx.syms) # dummy parse tree for testing print () basn = ellyConfiguration.baseSource + '/' filn = sys.argv[1] if len(sys.argv) > 1 else 'test' # which FSA definition to use ins = ellyDefinitionReader.EllyDefinitionReader(basn + filn + '.t.elly') print ( 'template test with' , '<' + filn + '>' ) if ins.error != None: print ( ins.error ) sys.exit(1) comp = None try: comp = CompoundTable(ctx.syms,ins) # load templates except ellyException.TableFailure: print ( 'no compound template table generated' ) sys.exit(1) comp.dump()
import os, stat print 'morphology test from sys.stdin' if not ellyConfiguration.morphologicalStemming: print 'Elly not configured for morphological analysis' sys.exit(1) mode = os.fstat(0).st_mode # sys.stdin file status intr = not stat.S_ISREG(mode) # flag for reading from keyboard sfil = sys.argv[1] if len(sys.argv) > 1 else 'default' pfil = sys.argv[2] if len(sys.argv) > 2 else sfil base = ellyConfiguration.baseSource + '/' sdfn = ellyDefinitionReader.EllyDefinitionReader(base + sfil + '.stl.elly') pdfn = ellyDefinitionReader.EllyDefinitionReader(base + pfil + '.ptl.elly') if sdfn.error != None or pdfn.error != None: print >> sys.stderr, 'suf error=', sdfn.error print >> sys.stderr, 'pre error=', pdfn.error try: inf = inflectionStemmerEN.InflectionStemmerEN() mor = MorphologyAnalyzer(sdfn, pdfn) except ellyException.TableFailure: print >> sys.stderr, 'initialization failed' sys.exit(1) mor.suff.infl = inf # set up root restoration
] class TL(TreeLogic): # have to override dummy methods """ subclass to override sequence() """ def __init__(self, inp): """ initialize """ super(TL, self).__init__(inp) def sequence(self, x): """ reverse a sequence """ return x[::-1] rdr = ellyDefinitionReader.EllyDefinitionReader(indta) tre = TL(rdr) # create tree with nonsense logic n1 = Node() # put something in tree to test the unit test itself n1.id = 10001 n2 = Node() n2.id = 10002 n3 = Node() n3.id = 10003 n4 = Node() n4.id = 10004 tre.indx['A'] = n1 # note that real tree will have only lowercase! n1.contn['B'] = n2 n2.contn['C'] = n3 n2.contn['D'] = n4 n3.actns = 'Act: -- 3' # fake actions
# # unit test # if __name__ == '__main__': import sys import ellyToken import ellyConfiguration import ellyDefinitionReader system = sys.argv[1] if len(sys.argv) > 1 else 'test' print 'system=', system base = ellyConfiguration.baseSource + '/' dfn = ellyDefinitionReader.EllyDefinitionReader(base + system + '.ptl.elly') if dfn.error != None: print >> sys.stderr, dfn.error sys.exit(1) n = dfn.linecount() print n, 'definition lines' if n == 0: sys.exit(0) try: pre = PrefixTreeLogic(dfn) except ellyException.TableFailure: print >> sys.stderr, 'cannot load prefix table' sys.exit(1) print 'pre=', pre
k = tre.relatedness(a.upper(), b.upper()) x = tre.intersection().name if k >= 0 else NOname print("relatedness(" + a + "," + b + ")=", end=' ') print(k, "intersection=", x) def tdump(tre): """ show hierarchy tree """ kyl = tre.index.keys() for ky in kyl: if ky == '^': continue r = tre.index[ky] print(ky, '(', r.name, ')', 'lvl=', r.level, '<', r.parent.name) filn = sys.argv[1] + '.h.elly' if len(sys.argv) > 1 else data inp = ellyDefinitionReader.EllyDefinitionReader(filn) if inp.error != None: print(inp.error, file=sys.stderr) sys.exit(1) elif filn != data: print('reading from file=', filn, file=sys.stderr) print(inp.linecount(), "lines read") try: ctre = ConceptualHierarchy(inp) except ellyException.TableFailure: print('could not load hierarchy', file=sys.stderr) sys.exit(1) if ctre.isEmpty(): print("tree building failed")