def map_DDT_tags_to_CST_terminals(self):
grammar = Grammar()
with open("DDT-to-CST-mapping.csv", "r") as file:
data = csv.reader(file)
for row in data:
gr = GrammarRule(row[1], [row[0]], 0)
grammar.count_rule(gr)
return grammar
def parse_treebank(self, treebank_filename):
# Reading and xml-parsing the DDT file
print ">>TREEBANK: Reading and xml-parsing the DDT file."
with open(treebank_filename, "r") as file:
treebank = BeautifulSoup(file.read())
print ">>LOG: Time spent is %s seconds" % self.log.time_since_last_check()
grammar = Grammar()
# Extracting grammar from sentences
print ">>TREEBANK: Extracting grammar from sentences."
for sentence in treebank.findAll('s'):
terminals = {}
nonterminals = {}
# extract terminals
terminals_xml = sentence.findAll('t')
for t in terminals_xml:
terminal = Terminal(t.get('id'), t.get('cat'), t.get('lemma'), t.get('word'))
terminals[terminal.id] = terminal
# extract nonterminals
nonterminals_xml = sentence.findAll('nt')
for nt in nonterminals_xml:
nonterminal = Nonterminal(nt.get('id'), nt.get('cat'), nt.get('lemma'), nt.get('word'))
edges = []
for e in nt.findAll('edge'):
edges.append([e.get('idref'), e.get('label')])
nonterminal.add_edges(edges)
nonterminals[nonterminal.id] = nonterminal
# Count edge --> nonterminals
for nt in nonterminals:
edge_startpoint_cat = nonterminals[nt].category
for e in nonterminals[nt].edges:
if e[0] in nonterminals:
edge_endpoint_cat = nonterminals[e[0]].category
else:
edge_endpoint_cat = terminals[e[0]].category
gr = GrammarRule(e[1], [edge_startpoint_cat, edge_endpoint_cat], 0)
grammar.count_rule(gr)
# Count nonterminal --> edges
for nt in nonterminals:
if len(nonterminals[nt].edges) > 1:
constituents = []
for e in nonterminals[nt].edges:
if e[1] != "--":
constituents.append(e[1])
gr = GrammarRule(nonterminals[nt].category, constituents, 0)
grammar.count_rule(gr)
print ">>TREEBANK: Done. Total runtime %s seconds" % (time.time() - self.log.starttime)
return grammar
def map_DDT_tags_to_CST_terminals():
grammar = Grammar()
with open("DDT-to-CST-mapping.csv", "r") as file:
data = csv.reader(file)
for row in data:
gr = GrammarRule(row[1], [row[0]])
grammar.rules[gr] = 1
return grammar
def new_treebank_parser(self, treebank_filename):
# Reading and xml-parsing the DDT file
print ">>TREEBANK: Reading and xml-parsing the DDT file."
test = False
if test: treebank_filename = "test_sentence"
with open(treebank_filename, "r") as file:
treebank = BeautifulSoup(file.read())
print ">>LOG: Time spent is %s seconds" % self.log.time_since_last_check()
grammar = Grammar()
# Extracting grammar from sentences
print ">>TREEBANK: Extracting grammar from sentences."
for sentence in treebank.findAll('s'):
terminals = {}
nonterminals = {}
# extract terminals
terminals_xml = sentence.findAll('t')
for t in terminals_xml:
terminal = Terminal(t.get('id'), t.get('cat'), t.get('lemma'), t.get('word'))
terminals[terminal.id] = terminal
# extract nonterminals
nonterminals_xml = sentence.findAll('nt')
for nt in nonterminals_xml:
nonterminal = Nonterminal(nt.get('id'), nt.get('cat'), nt.get('lemma'), nt.get('word'))
edges = []
for e in nt.findAll('edge'):
edges.append([e.get('idref'), e.get('label')])
nonterminal.add_edges(edges)
nonterminals[nonterminal.id] = nonterminal
for nt in nonterminals:
if len(nonterminals[nt].edges) > 1:
cat = nonterminals[nt].category + "P"
r = []
for e in nonterminals[nt].edges:
if e[1] != "--":
if len(nonterminals[e[0]].edges) > 1:
r.append(nonterminals[e[0]].category + "P")
else:
r.append(nonterminals[e[0]].category)
else:
r.append(terminals[e[0]].category)
gr = GrammarRule(cat, r, 0)
grammar.count_rule(gr)
return grammar
def main(): global input, grammar g = Grammar(grammar) g.parse() gotos = GotoGenerator(g) gotos.generate() gotos.display() g.first_follow.display() parsing_table = Table(g, gotos) parsing_table.generate() lr_parser = LR_Parser(g, parsing_table, input) lr_parser.parse()
def main(): from Grammar import Grammar grammar = """ S' := S S := A c A S := B c c B A := c A A := a B := c c B B := b """ grammar = """ S' := S S := q A B C A := a | b b D B := a | E C := b | E D := C | E """ # grammar = """ # S := a S a | b S b | D # D := b D' # D' := a a D' | E # # """ # grammar = """ # S := i X t S S' | a # S' := e S | E # X := b # """ g = Grammar(grammar) g.parse() print g.first_follow.first print g.first_follow.follow
def main(*argv):
global input, grammar
if len(argv[0]) == 2:
grammar = """"""
with open(argv[0][0], 'r') as content_file:
grammar = content_file.read()
input = argv[0][1]
g = Grammar(grammar)
g.parse()
gotos = GotoGenerator(g)
gotos.generate()
gotos.display()
g.first_follow.display()
parsing_table = Table(g, gotos)
parsing_table.generate()
lr_parser = LR_Parser(g, parsing_table, input)
lr_parser.parse()
def parse_json(json_input):
"""
:param string:
:return:
"""
# load const tree
const_tree = ConstTree.from_string(json_input['const_tree'])
# load grammar
grammar_list = [Grammar.from_string(i) for i in json_input['grammar']]
# load lexicon list
for i in json_input['lexicon']:
Lexicon.from_string(i)
word_set = Lexicon.lexicon_dict
root = generate_f_strcuture(const_tree, grammar_list, word_set)
output_string = f_structure_to_xml(json_input['const_tree'], root)
return output_string
def main():
exit = False
exit2 = False
while not exit:
file_name = raw_input("\n>Where is the Grammar =) ?: ")
print "\nGrammar:"
grammar = Grammar()
grammar.init_grammar(file_name)
print "\nNon Terminals: " + grammar.non_terminals.__str__()
print "\nTerminals:" + grammar.terminals.__str__()
grammar.define_first_sets()
grammar.define_follow_sets()
print "\nFirst and Follow Set of Productions:\n"
print_table_first_follow_set(grammar)
print "\nParse Table of the Grammar: \n"
parse_table = build_parse_table(grammar)
parse_table_with_sync_set(parse_table,grammar)
is_ambiguous = print_parse_table(parse_table, grammar)
if is_ambiguous:
print "The Grammar is Ambiguous"
else:
print "The Grammar is not Ambiguous"
while not exit2:
input_string = raw_input("\nString to Analyze: ")
if '$' in input_string:
ll_parsing(parse_table, grammar, input_string)
else:
print "\nYou must enter the end of string character ($)"
exit2 = raw_input("\nEnter another String to Analyze? ") != 'yes'
exit = raw_input("\nEnter another Grammar? ") != 'yes'
print "*"*100
import sys
sys.path.append('./IntermiddleClass')
from intermiddleClass import IntermiddleClass
from Grammar import Grammar
grammar = Grammar("gram2")
iClass = IntermiddleClass()
grammar.setNonTerminals("A")
grammar.setNonTerminals("B")
grammar.setTerminals("1")
grammar.setTerminals("0")
grammar.setInitialNT("A")
grammar.setProductions("A>0 B | A>1 A | A>epsilon")
grammar.setProductions("B>0 B | B>1 A")
# if (grammar.onlyEvaluate('0101')):
# print("Cadena válida")
# else:
# print("Cadena invalida")
print(grammar.evaluateString("0101"))
# print(iClass.transformAFD(grammar, '0101'))
while result[index][poz] in self.grammar.getTerminals():
newList.append(result[index][poz])
poz += 1
for elem1 in rhs:
newList.append(elem1)
for i in range(poz + 1, len(result[index])):
newList.append(result[index][i])
index += 1
result.append(newList)
return result
if __name__ == '__main__':
grammar1 = Grammar("g1.txt")
parser1 = Parser(grammar1)
string = 'a * ( a # a )'
sequence1 = string.split(" ")
se1 = string.split(" ")
print("Grammar 1:")
finish = parser1.parseSequence(se1)
print(finish)
print(parser1.derivationString(finish))
print(' ')
scanner2 = Scanner("pb1.txt")
scanner2.scan()
sequence2 = []
pif = scanner2.programInternalForm
for elem in pif:
elif tag == 'function':
statement = self._function(token, buffer)
else:
raise Exception('BUG: invalid token %s' % tag)
statements.append(statement)
return statements
def define_stmt(self, (tag, left, right, sublist), buffer):
name_tup, value_tup = sublist
value_tag = value_tup[0]
name = getString(name_tup, buffer)
value = getString(value_tup, buffer)
if value_tag == 'regex':
value = self._regex(value_tup, buffer)
elif value_tag == 'varname':
if not self.context.lexicon.has_key(value):
_error(buffer, value_tup[1], 'no such variable')
value = self.context.lexicon[value]
else:
raise Exception('BUG: invalid token %s' % value_tag)
self.context.lexicon[name] = value
def grammar_stmt(self, (tag, left, right, sublist), buffer):
map = singleMap(sublist)
grammar = Grammar()
grammar.name = getString(map['varname'], buffer)
grammar.statements = self._suite(map['suite'], buffer)
if map.has_key('inherit'):
grammar.inherit = self._inherit(map['inherit'], buffer)
self.context.grammars[grammar.name] = grammar
def __init__(self, config):
self.connection = Connection(config)
self.grammar = Grammar()
class Builder:
wheres = []
orders = []
tables = []
fields = []
groups = []
havings = []
limits = {}
distinct = False
def __init__(self, config):
self.connection = Connection(config)
self.grammar = Grammar()
"""
Select the table.
@param string table
@return self
"""
def table(self, table):
if self.connection.config.has_key('prefix'):
table = self.connection.config['prefix']+table
if not table in self.tables:
self.tables.append(table)
return self
"""
Add an "order by".
@param dict|string sort
@param string way
@return self
"""
def order(self, sort, way="asc"):
if isinstance(sort, dict):
for k, v in sort.items():
self.orders.append({'sort': k, 'way': v})
else:
self.orders.append({'sort': sort, 'way': way.lower()})
return self
"""
Select the fields.
@param tuple args
@return self
"""
def select(self, *args):
args = list(args)
if len(args) == 0:
self.fields = ["*"]
else:
for item in args:
self.fields += item if isinstance(item, list) else [item]
self.fields = list(set(self.fields))
return self
"""
Add a "distinct".
@return self
"""
def distinct(self):
self.distinct = True
return self
"""
Limit the number of results.
@param int offset
@param int num
@return self
"""
def limit(self, offset, num=0):
self.limits = {'num': offset, 'offset': 0} if num==0 else {'offset': offset, 'num': num}
return self
"""
Add a basic "where".
@param string field
@param string operator
@param string value
@param string boolean
@return self
"""
def where(self, field, operator=None, value=None, boolean='and'):
if not value:
value = operator
operator = '='
whereType = 'basic'
self.wheres.append({'field': field, 'value': value, 'operator': operator, 'boolean': boolean, 'type': whereType})
return self
"""
Add a "or where".
@param string field
@param string operator
@param string value
@return self
"""
def orWhere(self, field, operator=None, value=None):
return self.where(field, operator, value, 'or')
"""
Add a where between.
@param string field
@param string value
@param string boolean
@param boolean isNot
@return self
"""
def whereBetween(self, field, value, boolean='and', isNot=False):
if not isinstance(value, list):
value = list(value)
whereType = 'between'
self.wheres.append({'field': field, 'value': value, 'boolean': boolean, 'not': isNot, 'type': whereType})
return self
"""
Add a or where between.
@param string field
@param string value
@return self
"""
def orwhereBetween(self, field, value):
return self.whereBetween(field, value, 'or')
"""
Add a where not between.
@param string field
@param string value
@return self
"""
def whereNotBetween(self, field, value):
return self.whereBetween(field, value, 'and', True)
"""
Add a or where not between.
@param string field
@param string value
@return self
"""
def orWhereNotBetween(self, field, value):
return self.whereBetween(field, value, 'or', True)
"""
Add a where in.
@param string field
@param string value
@param string boolean
@param boolean isNot
@return self
"""
def whereIn(self, field, value, boolean='and', isNot=False):
if not isinstance(value, list):
value = list(value)
whereType = 'in'
self.wheres.append({'field': field, 'value': value, 'boolean': boolean, 'not': isNot, 'type': whereType})
return self
"""
Add a or where in.
@param string field
@param string value
@return self
"""
def orWhereIn(self, field, value):
return self.whereIn(field, value, 'or')
"""
Add a where not in.
@param string field
@param string value
@return self
"""
def whereNotIn(self, field, value):
return self.whereIn(field, value, 'and', True)
"""
Add a or where not in.
@param string field
@param string value
@return self
"""
def orWhereNotIn(self, field, value):
return self.whereIn(field, value, 'or', True)
"""
Add a where is null.
@param string field
@param string boolean
@param boolean isNot
@return self
"""
def whereNull(self, field, boolean='and', isNot=False):
whereType = 'null'
self.wheres.append({'field': field, 'boolean': boolean, 'not': isNot, 'type': whereType})
return self
"""
Add a or where is null.
@param string field
@return self
"""
def orWhereNull(self, field):
return self.whereNull(field, 'or')
"""
Add a where is not null.
@param string field
@return self
"""
def whereNotNull(self, field):
return self.whereNull(field, 'and', True)
"""
Add a or where is not null.
@param string field
@return self
"""
def orWhereNotNull(self, field):
return self.whereNull(field, 'or', True)
"""
Add a raw where.
@param string sql
#param string boolean
@return self
"""
def whereRaw(self, sql, boolean='and'):
whereType = 'raw'
self.wheres.append({'sql': sql, 'boolean': boolean, 'type': whereType})
return self
"""
Add a or raw where.
@param string sql
@return self
"""
def orWhereRaw(self, sql):
return self.whereRaw(sql, 'or')
"""
Add a group by.
@param tuple args
@return self
"""
def groupBy(self, *args):
args = list(args)
for item in args:
self.groups += item if isinstance(item, list) else [item]
self.groups = list(set(self.groups))
return self
"""
Add a having.
@param string field
@param string operator
@param string value
@param string boolean
@return self
"""
def having(self, field, operator=None, value=None, boolean='and'):
whereType = 'basic'
self.havings.append({'field': field, 'value': value, 'operator': operator, 'boolean': boolean, 'type': whereType})
return self
"""
Add a or having.
@param string field
@param string operator
@param string value
@return self
"""
def orHaving(self, field, operator=None, value=None):
return self.having(field, operator, value, 'or')
"""
Add a raw having.
@param string sql
@param string boolean
@return self
"""
def havingRaw(self, sql, boolean='and'):
whereType = 'raw'
self.havings.append({'sql': sql, 'boolean': boolean, 'type': whereType})
return self
"""
Add a raw or having.
@param string sql
@return self
"""
def orHavingRaw(self, sql):
return self.havingRaw(sql, 'or')
"""
def join(self):
return self.whereNull(field, 'or')
def leftJoin(self):
pass
"""
"""
get all of results
@return tuple
"""
def get(self):
results = self.connection.select(self.__toSql())
self.__optEnd()
return results
"""
get the first column of results
@return dict
"""
def first(self):
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0] if len(results) != 0 else results
"""
get the field from the first column of results
@param string field
@return mixed
"""
def pluck(self, field):
self.fields = [field]
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0][field] if len(results) != 0 else None
"""
get list of the field from results
"""
def lists(self, field):
self.fields = [field]
results = self.connection.select(self.__toSql())
self.__optEnd()
if len(results) == 0:
return None
else:
lists = []
for item in results:
lists.append(item[field])
return lists
"""
Return the sum of results of a given field.
@return numeric
"""
def count(self):
return self.aggregate(self.count.__name__, '*')
"""
Return the max of results of a given field.
@param string field
@return numeric
"""
def max(self, field):
return self.aggregate(self.max.__name__, field)
"""
Return the min of results of a given field.
@param string field
@return numeric
"""
def min(self, field):
return self.aggregate(self.min.__name__, field)
"""
Return the avg of results of a given field.
@param string field
@return numeric
"""
def avg(self, field):
return self.aggregate(self.avg.__name__, field)
"""
Return the sum of results of a given field.
@param string field
@return numeric
"""
def sum(self, field):
return self.aggregate(self.sum.__name__, field)
"""
Deal aggregate query.
@param string func
@param string field
@return numeric
"""
def aggregate(self, func, field):
self.fields = [func+"("+field+") as "+func]
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0][func] if len(results) != 0 else 0
"""
Insert a recode.
@return boolean
"""
def insert(self, values):
sql = self.grammar.compileInsert(self, values)
self.__optEnd()
return self.connection.insert(sql)
"""
Get the insert id
@return int
"""
def getInsertId(self):
return self.connection.getInsertId()
"""
Update a record.
@param dict values
@return boolean
"""
def update(self, values):
sql = self.grammar.compileUpdate(self, values)
self.__optEnd()
return self.connection.update(sql)
"""
Delete a record.
@return boolean
"""
def delete(self):
sql = self.grammar.compileDelete(self)
self.__optEnd()
return self.connection.delete(sql)
"""
Create the select sql
@return string
"""
def __toSql(self):
return self.grammar.compileSelect(self)
"""
clean up record
"""
def __optEnd(self):
self.wheres = []
self.orders = []
self.tables = []
self.fields = []
self.groups = []
self.havings = []
self.limits = {}
self.distinct = False
import CodeGenerator
from Grammar import Grammar
file = open("output/Grammar.txt", 'r')
data = file.read().replace(' ', '').replace('\t', '').replace('\n', ';')
file.close()
grammar = Grammar(data)
file = open('output/Output.py', 'w')
file.write(CodeGenerator.gen_basic())
for rule in grammar.get_rules():
file.write(CodeGenerator.gen_rule(rule))
file.write(CodeGenerator.gen_main())
file.close()
for i in dict[key]:
if key == i:
count -= 1
elif i in self.followDict.keys():
for elem in self.followDict[i]:
self.addToFollow(key, elem)
count = count - 1
else:
continue
if count == 0:
break
# remove duplicates
for elem in self.followDict:
l = list(dict.fromkeys(self.followDict[elem]))
self.followDict[elem] = l
if __name__ == '__main__':
grammar = Grammar("g3.txt")
parser = Parser(grammar)
parser.first()
parser.follow()
print("First")
print(parser.firstDict)
# print(len(parser.firstDict))
print("Follow")
print(parser.followDict)
def contains(self, word: str):
queue = Queue()
prods_list = [([self.start_symb], None)]
prods_list.append(([
self.
rename_map[f"[{self.tm.start_state}, #, {word[0]}, {word[0]}]"],
"A2"
], (["A1"], [
self.
rename_map[f"[{self.tm.start_state}, #, {word[0]}, {word[0]}]"],
"A2"
])))
tmp_sentence = [
self.
rename_map[f"[{self.tm.start_state}, #, {word[0]}, {word[0]}]"],
"A2"
]
for item in word[1:-1]:
tmp_sentence = tmp_sentence[:-1] + [
self.rename_map[f"[{item}, {item}]"], "A2"
]
prods_list.append(
(tmp_sentence, (["A2"],
[self.rename_map[f"[{item}, {item}]"], "A2"])))
tmp_sentence = tmp_sentence[:-1] + [
self.rename_map[f"[{word[-1]}, {word[-1]}, $]"]
]
prods_list.append(
(tmp_sentence,
(["A2"], [self.rename_map[f"[{word[-1]}, {word[-1]}, $]"]])))
queue.put((tmp_sentence, prods_list))
visited_sentences = []
while queue.qsize() > 0:
sentence, prods_consequence = queue.get()
sent_str = ",".join(sentence)
if sent_str in visited_sentences:
continue
else:
visited_sentences.append(sent_str)
for prod in self.productions:
head, body = prod
indexes = Grammar.get_subsequence(head, sentence)
if len(indexes) == 0:
continue
for ind_start, ind_end in indexes:
res_part1 = [sentence[i] for i in range(ind_start)]
res_part2 = body
res_part3 = [
sentence[i] for i in range(ind_end, len(sentence))
]
new_sentence = res_part1 + res_part2 + res_part3
new_prods_consequence = prods_consequence.copy()
new_prods_consequence.append((new_sentence, prod))
queue.put((new_sentence, new_prods_consequence))
if "".join(new_sentence) == word:
return new_sentence, new_prods_consequence
return False
def click(self):
gram=self.textEdit.toPlainText().split('\n')
sentence=self.lineEdit.text()
grammar=Grammar()
grammar.insert_from_arr(gram)
if grammar.have_left_recursion()==True:
grammar.eliminating_left_recursion()
if grammar.have_left_factor()==True:
grammar.left_factoring()
follow=grammar.follow()
if grammar.is_LL1()==False:
print('不是LL(1)文法')
return
table=grammar.consuct_predictive_parsing_table(follow)
step = grammar.predictive_parsing(table, sentence)
for i in range(len(step)):
self.tableWidget.insertRow(self.tableWidget.rowCount())
for i in range(self.tableWidget.rowCount()):
for j in range(self.tableWidget.columnCount()):
self.tableWidget.setItem(i,j,QTableWidgetItem(step[i][j]))
#把分析过程写入Excel文件中
from pyexcelerate import Workbook
data = step # data is a 2D array
wb = Workbook()
wb.new_sheet("step", data=data)
wb.save("output.xlsx")
def __init__(self):
self.table = Table()
self.grammar = Grammar()
#setup as constructor
self.setup()
from FiniteAutomaton import FiniteAutomaton
from Grammar import Grammar
if __name__ == '__main__':
# read the things from the file
print("1.Grammar from the requested file -------------------------------")
grammar = Grammar.read_from_file('rg1.txt')
# grammar = Grammar.read_from_console()
print(grammar)
print("----------------------------------------------------------")
print("2.Production for the grammar and no_terminal A ------------------")
grammar.get_productions_for('A')
grammar.show_productions_for('A')
print("----------------------------------------------------------")
print("3.Finite Automata ------------------")
finiteAutomata = FiniteAutomaton.read_from_file('fa1.txt')
# finiteAutomata = FiniteAutomaton.read_from_console()
print(finiteAutomata)
print("----------------------------------------------------------")
print(
"3.Is regular and then compute Fa-----------------------------------------"
)
grammar = Grammar.read_from_file('rg1.txt')
# grammar = Grammar.read_from_console()
class Parser:
def __init__(self, grammarFileName, givenParserOutputName):
self.currentState = 'q'
self.index = 0
self.workingStack = []
self.grammar = Grammar(grammarFileName)
self.parserOutput = ParserOutput(givenParserOutputName, self.grammar)
self.debug = True
self.inputStack = []
self.epsilonCount = 0
self.derivationsString = ""
def printAll(self):
print("-----------------------------------------------------------------------------------")
print("Current state: " + self.currentState)
print("Current index: " + str(self.index))
print("Working stack: " + str(self.workingStack))
print("Input stack: " + str(self.inputStack))
def expand(self):
if isinstance(self.inputStack[0], tuple):
nonterminal = self.inputStack.pop(0)
elif isinstance(self.inputStack[0], list):
nonterminal = (self.inputStack[0].pop(0), 1)
if len(self.inputStack[0]) == 0:
self.inputStack.pop(0)
else:
nonterminal = (self.inputStack.pop(0), 1)
self.workingStack.append(nonterminal)
newProduct = self.grammar.getProductions()[nonterminal]
self.inputStack.insert(0, newProduct)
def advance(self):
self.index += 1
if isinstance(self.inputStack[0], tuple):
terminal = self.inputStack[0].pop(0)
elif isinstance(self.inputStack[0], list):
terminal = self.inputStack[0].pop(0)
if len(self.inputStack[0]) == 0:
self.inputStack.pop(0)
else:
terminal = self.inputStack.pop(0)
self.workingStack.append(terminal)
def momentaryInsucces(self):
self.currentState = 'b'
def back(self):
self.index -= 1
lastFromWorkingStack = [self.workingStack.pop()]
if lastFromWorkingStack == ['epsilon']:
self.epsilonCount -= 1
self.inputStack.insert(0, lastFromWorkingStack)
def anotherTry(self):
lastFromWorkingStack = self.workingStack.pop()
checkIfNextExists = (lastFromWorkingStack[0], lastFromWorkingStack[1] + 1)
if checkIfNextExists in self.grammar.getProductions():
self.currentState = 'q'
self.workingStack.append(checkIfNextExists)
removedElements = 0
while removedElements < len(self.grammar.getProductions()[lastFromWorkingStack]):
if isinstance(self.inputStack[0], list):
if len(self.inputStack[0]) == 1:
self.inputStack.pop(0)
removedElements += 1
else:
while len(self.inputStack[0]) > 0 and removedElements < \
len(self.grammar.getProductions()[lastFromWorkingStack]):
self.inputStack[0].pop(0)
removedElements += 1
if len(self.inputStack[0]) == 0:
self.inputStack.pop(0)
else:
self.inputStack.pop(0)
removedElements += 1
self.inputStack.insert(0, self.grammar.getProductions()[checkIfNextExists])
elif self.index == 0 and lastFromWorkingStack[0] == self.grammar.getInitialNonTerminal():
self.currentState = 'e'
else:
removedElements = 0
while removedElements < len(self.grammar.getProductions()[lastFromWorkingStack]):
if isinstance(self.inputStack[0], list):
if len(self.inputStack[0]) == 1:
self.inputStack.pop(0)
removedElements += 1
else:
while len(self.inputStack[0]) > 0 and removedElements < \
len(self.grammar.getProductions()[lastFromWorkingStack]):
self.inputStack[0].pop(0)
removedElements += 1
if len(self.inputStack[0]) == 0:
self.inputStack.pop(0)
else:
self.inputStack.pop(0)
removedElements += 1
self.inputStack.insert(0, [lastFromWorkingStack[0]])
def success(self):
self.currentState = 'f'
self.index += 1
def checkWordLength(self, w):
if len(w) > self.index - self.epsilonCount:
if self.inputStack[0][0] == 'epsilon':
self.epsilonCount += 1
return True
if self.inputStack[0][0] in self.grammar.getNonTerminals():
return True
return self.inputStack[0][0] == w[self.index - self.epsilonCount]
return False
def addOneStepDerivationString(self):
oneStepDerivationString = "(" + self.currentState + ", " + str(self.index) + ", " + str(self.workingStack) + \
", " + str(self.inputStack) + ")\n"
self.derivationsString += oneStepDerivationString
def runAlgorithm(self, w):
self.currentState = 'q'
self.index = 0
self.workingStack = []
self.debug = True
self.inputStack = [(self.grammar.getInitialNonTerminal(), 1)]
self.epsilonCount = 0
self.derivationsString = ""
while self.currentState != "f" and self.currentState != "e":
if self.debug:
self.printAll()
self.addOneStepDerivationString()
if self.currentState == "q":
if len(self.inputStack) == 0 and self.index - self.epsilonCount == len(w):
self.derivationsString += "|- succ"
self.currentState = "f"
else:
if len(self.inputStack) > 0 and self.inputStack[0][0] in self.grammar.getNonTerminals():
self.derivationsString += "|- exp"
self.expand()
elif len(self.inputStack) > 0 and self.inputStack[0][0] in self.grammar.getTerminals() \
and self.checkWordLength(w):
self.derivationsString += "|- adv"
self.advance()
else:
self.derivationsString += "|- mi"
self.momentaryInsucces()
elif self.currentState == "b":
if self.workingStack[-1] in self.grammar.getTerminals():
self.derivationsString += "|- bk"
self.back()
else:
self.derivationsString += "|- at"
self.anotherTry()
self.addOneStepDerivationString()
if self.currentState == "e":
print("Error")
self.parserOutput.writeRepresentationsToFile("error")
else:
print("Finished")
self.parserOutput.setResultAndCalculateProductionString(self.workingStack, self.derivationsString)
self.parserOutput.writeRepresentationsToFile("success")
def transformGrammar(self, afd, string):
non_terminals = afd.states
terminals = afd.alphabet
initial_nt = afd.initialState
productions = afd.transitions
epsilon_prod = ""
msg = ""
for item in afd.getAcceptanceStates():
prod = {'fS': item, 't': 'epsilon', 'lS': ""}
# epsilon_prod = f"{item}>epsilon"
productions.append(prod)
for item in productions:
item["String"] = f"{item['fS']}>{item['t']} {item['lS']}"
grammar = Grammar(afd.getName())
for nt in non_terminals:
grammar.setNonTerminals(nt)
for t in terminals:
grammar.setTerminals(t)
grammar.setInitialNT(initial_nt)
for p in productions:
grammar.setProductions(p['String'])
grammarExtended = grammar.evaluateString(string)
msg = f"{grammarExtended}"
return msg
class Parser:
def __init__(self, fileName):
self.__grammar = Grammar(fileName)
self.__firstSet = {}
self.__followSet = {}
self.generate_first_set()
self.__parsing_table = {}
print(self.__grammar.get_productions())
def generate_first_set(self):
for nonterminal in self.__grammar.get_nonterminals():
first = self.first_of_nonterminal(nonterminal)
self.__firstSet[nonterminal] = first
def first_of_nonterminal(self, nonterminal):
firstSet = []
if nonterminal in self.__firstSet.keys():
return self.__firstSet[nonterminal]
productions = self.__grammar.get_productions()
for rule in productions[nonterminal]:
i = 0
firstSymbol = rule[i]
while i < len(rule) - 1:
if firstSymbol == "epsilon":
i = i + 1
firstSymbol = rule[i]
else:
break
if i == len(rule) - 1 and firstSymbol == "epsilon":
firstSet.append("epsilon")
elif firstSymbol in self.__grammar.get_terminals():
firstSet.append(firstSymbol)
else:
firstSet.extend(self.first_of_nonterminal(firstSymbol))
return firstSet
def get_first_set(self):
return self.__firstSet
def generate_follow_set(self):
for nonterminal in self.__grammar.get_nonterminals():
follow = self.follow_of(nonterminal)
self.__followSet[nonterminal] = follow
print(self.__followSet)
def follow_of(self, nonterminal):
followSet = []
productions = self.__grammar.get_productions()
if nonterminal in self.__followSet.keys():
return self.__followSet[nonterminal]
if nonterminal == self.__grammar.get_start():
followSet.append("$")
for production in productions.items():
for elem in production[1]:
for i in range(len(elem)):
if elem[i] == nonterminal:
if i < len(elem) - 1:
if elem[i + 1] in self.__grammar.get_terminals():
followSet.append(elem[i + 1])
return followSet
if elem[i +
1] in self.__grammar.get_nonterminals():
followSet.extend(self.__firstSet[elem[i + 1]])
if "epsilon" in self.__firstSet[elem[i + 1]]:
followSet.extend(
self.__followSet[production[0]])
for i in range(len(followSet)):
if followSet[i] == "epsilon":
followSet.pop(i)
break
return followSet
else:
followSet.extend(self.follow_of(production[0]))
return followSet
return followSet
def get_follow_set(self):
return self.__followSet
def create_table(self):
nonterminals = self.__grammar.get_nonterminals()
terminals = self.__grammar.get_terminals()
terminals.append("$")
productions = self.__grammar.get_productions()
productionIndex = 1
for entry in productions.items():
for production in entry[1]:
if production[0] in terminals:
self.__parsing_table[(entry[0],
production[0])] = productionIndex
elif production[0] == "epsilon":
follow = self.__followSet[entry[0]]
for elem in follow:
self.__parsing_table[(entry[0],
elem)] = productionIndex
elif production[0] in nonterminals:
first = self.__firstSet[entry[0]]
for elem in first:
self.__parsing_table[(entry[0],
elem)] = productionIndex
productionIndex += 1
for cell in self.__parsing_table.items():
print(cell)
config.workStack.pop(-1)
config.workStack.append(nextProd)
config.inputStack = config.inputStack[
len(lastProduction[1]):]
config.inputStack = nextProd[1] + config.inputStack
elif config.index == 0 and lastProduction[0] == g.S:
config.state = State.ERROR
else:
config.workStack.pop(-1)
if lastProduction[1] == ['e']:
config.inputStack = [lastProduction[0]
] + config.inputStack
else:
config.inputStack = [
lastProduction[0]
] + config.inputStack[len(lastProduction[1]):]
if config.state == State.ERROR:
print("Error")
else:
for prod in config.workStack:
if prod in g.P:
print(prod)
g = Grammar.read_from_file('input.txt')
input = ['2', '4', '11', '12', '15', '30', '1', '17', '16']
# input='aacbc'
f = open("in.txt", "r")
input = f.readline().split()
parser(g, input)
print "\n\n"
self.gram.add_production(start, partialseq)
partialseq = []
print "START: " + start + "\nSequence:"
for x in partialseq:
print x,
print "\n\n"
self.gram.add_production(start, tuple(partialseq))
def readFromFile(self):
f = open('../bnf2')
for line in f:
self.stream += line
grammar = Grammar()
sc = Importer(grammar)
for token in sc.parse():
pass
sc.readProductions()
grammar.set_start_symbol('<program>')
rep = Repository("../sample.cpp")
terminals = []
for token in rep.tokenize():
if token.code > 49 or token.code == 1:
terminals.append(token.type)
else:
terminals.append(token.value)
def parser(g: Grammar, sequence):
config = Configuration(g.S)
while config.state != State.FINAL and config.state != State.ERROR:
if config.state == State.NORMAL:
if len(config.inputStack) == 0 and config.index == len(sequence):
config.state = State.FINAL
elif len(config.inputStack) == 0:
config.state = State.ERROR
else:
if config.inputStack[0] in g.N:
nonTerminal = config.inputStack[0]
firstProduction = g.get_productions_for_non_terminal(
nonTerminal)[0]
config.workStack.append(firstProduction)
config.inputStack = firstProduction[1] + config.inputStack[
1:]
else:
if config.index == len(sequence):
config.state = State.BACK
elif config.inputStack[0] == 'e':
config.workStack.append('e')
config.inputStack = config.inputStack[1:]
elif config.inputStack[0] == sequence[config.index]:
config.index += 1
config.workStack.append(config.inputStack[0])
config.inputStack = config.inputStack[1:]
else:
config.state = State.BACK
else:
if config.state == State.BACK:
if config.workStack[-1] in g.E:
if config.workStack[-1] == 'e':
config.workStack.pop(-1)
else:
config.index -= 1
terminal = config.workStack.pop(-1)
config.inputStack = [terminal] + config.inputStack
else:
lastProduction = config.workStack[-1]
productions = g.get_productions_for_non_terminal(
lastProduction[0])
nextProd = getNextProduction(lastProduction, productions)
if nextProd:
config.state = State.NORMAL
config.workStack.pop(-1)
config.workStack.append(nextProd)
config.inputStack = config.inputStack[
len(lastProduction[1]):]
config.inputStack = nextProd[1] + config.inputStack
elif config.index == 0 and lastProduction[0] == g.S:
config.state = State.ERROR
else:
config.workStack.pop(-1)
if lastProduction[1] == ['e']:
config.inputStack = [lastProduction[0]
] + config.inputStack
else:
config.inputStack = [
lastProduction[0]
] + config.inputStack[len(lastProduction[1]):]
if config.state == State.ERROR:
print("Error")
else:
for prod in config.workStack:
if prod in g.P:
print(prod)
def __init__(self):
self.wiki = Wikipedia()
self.translate = Translate()
self.grammar = Grammar()
class Parser:
def __init__(self, ):
self.gram = Grammar()
self.look_ahead = 1
self.stack = ["$"] #list stack, append pushes,
self._non_terms = set()
self._terms = set("$")
self.parsetable = self.resetParsetable(
) #dict of {non_term:{term:(first,follow)}
self.firstsets = {}
self.followsets = {}
def setGrammar(self, gram):
self.gram = gram
self.setNonTerms()
self.setTerms()
self.stack = ['$', 'S']
self.buildFirstsets()
self.buildFollowsets()
self.populateParsetable()
def setLook_ahead(self, k):
self.look_ahead = 1
print("Look ahead currently limited to 1")
def describe(self):
print("\nThis is a LL(%d) shift-reduce grammar parser" %
(self.look_ahead))
if self.gram == None:
print("No grammar set")
return
print("It is using the following grammar: ")
print(self.gram)
self.describeParse()
#self.describeStack()
def describeParse(self):
print("Parsetable:", self.parsetable)
print("")
def describeStack(self):
print("Current Stack:")
print("bottom ", self.stack, " top")
input()
def resetStack(self):
self.stack = ["$", "S"]
def setNonTerms(self):
self._non_terms = set()
for rule in self.gram.getRules():
self._non_terms.add(rule.getVar())
def setTerms(self):
self._terms = set('$')
for rule in self.gram.getRules():
for yld in rule.getYield():
self._terms = self._terms | set(char for char in yld)
self._terms = self._terms - self._non_terms - set("/")
def resetParsetable(self):
inner = dict(
zip(self._non_terms,
[set().copy() for i in range(len(self._non_terms))]))
inn_cop = [deepcopy(inner) for i in range(len(self._terms))]
self.parsetable = dict(zip(self._terms, inn_cop))
#dict of {term:{non_term:(rule_number)}
def populateParsetable(self):
self.resetParsetable()
#`print("")
#print("First sets: " , self.firstsets)
#print("Followsets: " , self.followsets)
keys_considered = set()
for non_term in self._non_terms:
for term in self.firstsets[non_term]: #term is the keys
if term == "/":
for _term in self.followsets[non_term]:
self.parsetable[_term][non_term] = self.parsetable[
_term][non_term] | self.firstsets[non_term][term]
else:
self.parsetable[term][non_term] = self.parsetable[term][
non_term] | self.firstsets[non_term][term]
return
def buildFollowsets(self):
self.followsets = {}
for rule in self.gram.getRules():
self.followsets[rule.getVar()] = set()
nextset = self.followset('S')
while nextset != self.followsets:
self.followsets = nextset
nextset = self.followset('S')
def followset(self, symbol):
nextset = deepcopy(self.followsets)
if symbol == "S":
nextset['S'].add('$')
for rule_i in range(1, self.gram.getLength() + 1):
rule = self.gram[rule_i]
yld = rule.getYield()[0]
for i in range(len(yld)):
char = yld[i]
if char in self._non_terms:
if i + 1 == len(yld) and char != rule.getVar():
nextset[char] = nextset[char] | nextset[rule.getVar()]
break
else:
remove = yld[i + 1] in self.firstsets
if "/" in self.firstSet(yld[i + 1]):
nextset[char] = nextset[char] | nextset[
rule.getVar()]
nextset[char] = nextset[char] | set(
self.firstSet(yld[i + 1])) - set(['/'])
#if remove:
# del self.firstsets[yld[i+1]]
return nextset
def buildFirstsets(self): #builds the greater set of followsets
self.firstsets = {}
for rule in self.gram.getRules():
self.firstsets[rule.getVar()] = self.firstSet(rule.getVar())
#print(self.firstsets)
def firstSet(self, symbol):
#print("Called on ", symbol)
if symbol in self.firstsets: #base case 1
return self.firstsets[symbol]
if symbol not in self._non_terms: #base case 2
self.firstsets[symbol] = {
symbol: set([False])
} #dont come from a rule, just symbol
return self.firstsets[symbol]
_range = self.gram.getRuleRange(
symbol
) #get range as a set of integers where rule starts with symbol
firstS = dict() #empty dictionary
#print("")
#print(symbol,_range)
for rule_i in range(_range[0], _range[1]):
rule = self.gram[rule_i]
# print("Considering: ", rule)
yld = rule.getYield()[0]
for key in self.firstSet(yld[0]):
# print("Adding " , key, "to", symbol)
if key not in firstS:
firstS[key] = set()
firstS[key].add(rule_i)
return firstS
def preParse(self):
self.gram.removeRecursion() #currently just removes redundancy
for top in self.parsetable:
for sub in self.parsetable[top]:
if len(self.parsetable[top][sub]) > 1:
return False
return True
def parse(self, string):
if not self.preParse():
print(
"WARNING, THIS GRAMMAR IS NOT LL[1] AND/OR RECURSIVE AND SO THIS IS LIKELY INNACURATE OR NON-TERMINATING"
)
#self.describeParse()
self.describe()
string += "$"
self.resetStack()
stack = self.stack
ops = []
i = 0
print("Parsing", string)
input()
while i < len(string):
char = string[i]
print("Remaining String: ", string[i:])
self.describeStack()
#sleep(1)
if char == stack[-1]:
print("Reducing Stack, read: ", string[i], "popping: ",
stack.pop())
i += 1
else:
try:
rule_i = list(self.parsetable[char][stack.pop()])[0]
except (IndexError, ValueError, KeyError):
print("No valid rule. No shift/reduce possible")
print("\n\nRejected: ", string[:-1], "\n\n")
return False
ops.append(rule_i)
print("Rule ", rule_i, "matches: ", str(self.gram[rule_i]))
print("Shifting Stack, Pushing: ",
self.gram[rule_i].getYield()[0])
for t in self.gram[rule_i].getYield()[0][::-1]:
if t == "/":
break
else:
stack.append(t)
#print("Pushing: ", t)
#sleep(1.5)
input()
if stack == []:
print("\n\nAccepted: ", string[:-1], "\n\n")
print("Transitions: ", ops)
return True
else:
print("\n\nRejected: ", string[:-1], "\n\n")
return False
#!/usr/bin/env python
from Grammar import Grammar
from Grammar import Parser
import cgi
import cgitb
cgitb.enable()
form = cgi.FieldStorage()
rules = form['rules'].value.split('\r\n')
cmd = form['cmd'].value
g = Grammar()
p = Parser(g)
p.parse_rules(rules)
if cmd == 'generate':
print(g.derive('S'))
elif cmd == 'cnf':
print(g.to_cnf())
from Parser import Parser
from Grammar import Grammar
#P = Parser()
#G = Grammar()
#G.readGrammar('gram1.txt')
#P.setGrammar(G)
#for string in ['0','01','11','0011','10']:
# P.parse(string)
# input()
P = Parser()
G = Grammar()
G.readGrammar('gram3.txt')
P.setGrammar(G)
for string in['()', '(a+a)']: #['(a+a)','a','a+a','((a+a)+a)']:
P.parse(string)
input()
class Builder:
wheres = []
orders = []
tables = []
fields = []
groups = []
havings = []
limits = {}
distinct = False
def __init__(self, config):
self.connection = Connection(config)
self.grammar = Grammar()
"""
Select the table.
@param string table
@return self
"""
def table(self, table):
if self.connection.config.has_key('prefix'):
table = self.connection.config['prefix'] + table
if not table in self.tables:
self.tables.append(table)
return self
"""
Add an "order by".
@param dict|string sort
@param string way
@return self
"""
def order(self, sort, way="asc"):
if isinstance(sort, dict):
for k, v in sort.items():
self.orders.append({'sort': k, 'way': v})
else:
self.orders.append({'sort': sort, 'way': way.lower()})
return self
"""
Select the fields.
@param tuple args
@return self
"""
def select(self, *args):
args = list(args)
if len(args) == 0:
self.fields = ["*"]
else:
for item in args:
self.fields += item if isinstance(item, list) else [item]
self.fields = list(set(self.fields))
return self
"""
Add a "distinct".
@return self
"""
def distinct(self):
self.distinct = True
return self
"""
Limit the number of results.
@param int offset
@param int num
@return self
"""
def limit(self, offset, num=0):
self.limits = {
'num': offset,
'offset': 0
} if num == 0 else {
'offset': offset,
'num': num
}
return self
"""
Add a basic "where".
@param string field
@param string operator
@param string value
@param string boolean
@return self
"""
def where(self, field, operator=None, value=None, boolean='and'):
if not value:
value = operator
operator = '='
whereType = 'basic'
self.wheres.append({
'field': field,
'value': value,
'operator': operator,
'boolean': boolean,
'type': whereType
})
return self
"""
Add a "or where".
@param string field
@param string operator
@param string value
@return self
"""
def orWhere(self, field, operator=None, value=None):
return self.where(field, operator, value, 'or')
"""
Add a where between.
@param string field
@param string value
@param string boolean
@param boolean isNot
@return self
"""
def whereBetween(self, field, value, boolean='and', isNot=False):
if not isinstance(value, list):
value = list(value)
whereType = 'between'
self.wheres.append({
'field': field,
'value': value,
'boolean': boolean,
'not': isNot,
'type': whereType
})
return self
"""
Add a or where between.
@param string field
@param string value
@return self
"""
def orwhereBetween(self, field, value):
return self.whereBetween(field, value, 'or')
"""
Add a where not between.
@param string field
@param string value
@return self
"""
def whereNotBetween(self, field, value):
return self.whereBetween(field, value, 'and', True)
"""
Add a or where not between.
@param string field
@param string value
@return self
"""
def orWhereNotBetween(self, field, value):
return self.whereBetween(field, value, 'or', True)
"""
Add a where in.
@param string field
@param string value
@param string boolean
@param boolean isNot
@return self
"""
def whereIn(self, field, value, boolean='and', isNot=False):
if not isinstance(value, list):
value = list(value)
whereType = 'in'
self.wheres.append({
'field': field,
'value': value,
'boolean': boolean,
'not': isNot,
'type': whereType
})
return self
"""
Add a or where in.
@param string field
@param string value
@return self
"""
def orWhereIn(self, field, value):
return self.whereIn(field, value, 'or')
"""
Add a where not in.
@param string field
@param string value
@return self
"""
def whereNotIn(self, field, value):
return self.whereIn(field, value, 'and', True)
"""
Add a or where not in.
@param string field
@param string value
@return self
"""
def orWhereNotIn(self, field, value):
return self.whereIn(field, value, 'or', True)
"""
Add a where is null.
@param string field
@param string boolean
@param boolean isNot
@return self
"""
def whereNull(self, field, boolean='and', isNot=False):
whereType = 'null'
self.wheres.append({
'field': field,
'boolean': boolean,
'not': isNot,
'type': whereType
})
return self
"""
Add a or where is null.
@param string field
@return self
"""
def orWhereNull(self, field):
return self.whereNull(field, 'or')
"""
Add a where is not null.
@param string field
@return self
"""
def whereNotNull(self, field):
return self.whereNull(field, 'and', True)
"""
Add a or where is not null.
@param string field
@return self
"""
def orWhereNotNull(self, field):
return self.whereNull(field, 'or', True)
"""
Add a raw where.
@param string sql
#param string boolean
@return self
"""
def whereRaw(self, sql, boolean='and'):
whereType = 'raw'
self.wheres.append({'sql': sql, 'boolean': boolean, 'type': whereType})
return self
"""
Add a or raw where.
@param string sql
@return self
"""
def orWhereRaw(self, sql):
return self.whereRaw(sql, 'or')
"""
Add a group by.
@param tuple args
@return self
"""
def groupBy(self, *args):
args = list(args)
for item in args:
self.groups += item if isinstance(item, list) else [item]
self.groups = list(set(self.groups))
return self
"""
Add a having.
@param string field
@param string operator
@param string value
@param string boolean
@return self
"""
def having(self, field, operator=None, value=None, boolean='and'):
whereType = 'basic'
self.havings.append({
'field': field,
'value': value,
'operator': operator,
'boolean': boolean,
'type': whereType
})
return self
"""
Add a or having.
@param string field
@param string operator
@param string value
@return self
"""
def orHaving(self, field, operator=None, value=None):
return self.having(field, operator, value, 'or')
"""
Add a raw having.
@param string sql
@param string boolean
@return self
"""
def havingRaw(self, sql, boolean='and'):
whereType = 'raw'
self.havings.append({
'sql': sql,
'boolean': boolean,
'type': whereType
})
return self
"""
Add a raw or having.
@param string sql
@return self
"""
def orHavingRaw(self, sql):
return self.havingRaw(sql, 'or')
"""
def join(self):
return self.whereNull(field, 'or')
def leftJoin(self):
pass
"""
"""
get all of results
@return tuple
"""
def get(self):
results = self.connection.select(self.__toSql())
self.__optEnd()
return results
"""
get the first column of results
@return dict
"""
def first(self):
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0] if len(results) != 0 else results
"""
get the field from the first column of results
@param string field
@return mixed
"""
def pluck(self, field):
self.fields = [field]
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0][field] if len(results) != 0 else None
"""
get list of the field from results
"""
def lists(self, field):
self.fields = [field]
results = self.connection.select(self.__toSql())
self.__optEnd()
if len(results) == 0:
return None
else:
lists = []
for item in results:
lists.append(item[field])
return lists
"""
Return the sum of results of a given field.
@return numeric
"""
def count(self):
return self.aggregate(self.count.__name__, '*')
"""
Return the max of results of a given field.
@param string field
@return numeric
"""
def max(self, field):
return self.aggregate(self.max.__name__, field)
"""
Return the min of results of a given field.
@param string field
@return numeric
"""
def min(self, field):
return self.aggregate(self.min.__name__, field)
"""
Return the avg of results of a given field.
@param string field
@return numeric
"""
def avg(self, field):
return self.aggregate(self.avg.__name__, field)
"""
Return the sum of results of a given field.
@param string field
@return numeric
"""
def sum(self, field):
return self.aggregate(self.sum.__name__, field)
"""
Deal aggregate query.
@param string func
@param string field
@return numeric
"""
def aggregate(self, func, field):
self.fields = [func + "(" + field + ") as " + func]
results = self.connection.select(self.__toSql())
self.__optEnd()
return results[0][func] if len(results) != 0 else 0
"""
Insert a recode.
@return boolean
"""
def insert(self, values):
sql = self.grammar.compileInsert(self, values)
self.__optEnd()
return self.connection.insert(sql)
"""
Get the insert id
@return int
"""
def getInsertId(self):
return self.connection.getInsertId()
"""
Update a record.
@param dict values
@return boolean
"""
def update(self, values):
sql = self.grammar.compileUpdate(self, values)
self.__optEnd()
return self.connection.update(sql)
"""
Delete a record.
@return boolean
"""
def delete(self):
sql = self.grammar.compileDelete(self)
self.__optEnd()
return self.connection.delete(sql)
"""
Create the select sql
@return string
"""
def __toSql(self):
return self.grammar.compileSelect(self)
"""
clean up record
"""
def __optEnd(self):
self.wheres = []
self.orders = []
self.tables = []
self.fields = []
self.groups = []
self.havings = []
self.limits = {}
self.distinct = False
def read_from_file(input):
start = None
V = set()
T = set()
P = dict()
prodArray = []
with open('{}'.format(input)) as f:
line = f.readline().strip().split(' ')
V = set(line)
start = line[0]
line = f.readline().strip().split(' ')
T = set(line)
line = f.readline().strip()
while line != "":
i = 0
j = 1
tokens = []
last = len(line) + 1
while j != last:
sub = line[i:j]
if sub == " ":
i += 1
j += 1
continue
if sub in V or sub in T:
while sub in V or sub in T:
j += 1
sub = line[i:j]
if j == last: break
j -= 1
tokens.append(line[i:j])
i = j
j = i + 1
continue
if sub == ":":
j += 1
sub = line[i:j]
if sub == ":=":
tokens.append(sub)
i = j
j = i + 1
continue
if sub == "|":
tokens.append("|")
i += 1
j += 1
continue
prodArray.append(tokens)
line = f.readline().strip()
P = {v: [] for v in V}
for prod in prodArray:
v = prod[0]
currentProd = []
for i in range(2, len(prod)):
if prod[i] != "|":
currentProd.append(prod[i])
else:
P[v].append(currentProd)
currentProd = []
P[v].append(currentProd)
return Grammar(V, T, start, P)
#!/usr/bin/env python
from Grammar import Grammar
from Grammar import Parser
import cgi
import cgitb
cgitb.enable()
form = cgi.FieldStorage()
rules = form['rules'].value.split('\r\n')
cmd = form['cmd'].value
g = Grammar()
p = Parser(g)
p.parse_rules(rules)
if cmd == 'generate':
print(g.derive('S'))
elif cmd == 'cnf':
print(g.to_cnf())
from Parser import Parser
from Grammar import Grammar
grammar = Grammar("g2.txt")
grammar.printAll()
#grammar.printOneNonTerminal()
parser = Parser()
parser.runAlgorithm(['a', 'a', 'c', 'b', 'c'])
def run(self):
print ">>TREEBANK: Reading and xml-parsing the DDT file."
with open("ddt-1.0.xml", "r") as file:
treebank = BeautifulSoup(file.read(), "xml")
print ">>LOG: Time spent is %s seconds" % self.log.time_since_last_check()
counter = 0
grammar = Grammar()
print ">>TREEBANK: Extracting grammar from sentences."
for sentence in treebank.findAll("s"):
terminals = {}
nonterminals = {}
# extract terminals
terminals_xml = sentence.findAll("t")
for t in terminals_xml:
terminal = Terminal(t.get("id"), t.get("cat"), t.get("lemma"), t.get("word"))
terminals[terminal.id] = terminal
# extract nonterminals
nonterminals_xml = sentence.findAll("nt")
for nt in nonterminals_xml:
nonterminal = Nonterminal(nt.get("id"), nt.get("cat"), nt.get("lemma"), nt.get("word"))
edges = []
for e in nt.findAll("edge"):
edges.append([e.get("idref"), e.get("label")])
nonterminal.add_edges(edges)
nonterminals[nonterminal.id] = nonterminal
# Count edge --> nonterminals
for nt in nonterminals:
edge_startpoint_cat = nonterminals[nt].category
for e in nonterminals[nt].edges:
if e[0] in nonterminals:
edge_endpoint_cat = nonterminals[e[0]].category
else:
edge_endpoint_cat = terminals[e[0]].category
gr = GrammarRule(e[1], [edge_startpoint_cat, edge_endpoint_cat])
grammar.count_rule(gr)
# Count nonterminal --> edges
for nt in nonterminals:
if len(nonterminals[nt].edges) > 1:
constituents = []
for e in nonterminals[nt].edges:
if e[1] != "--":
constituents.append(e[1])
gr = GrammarRule(nonterminals[nt].category, constituents)
grammar.count_rule(gr)
print ">>LOG: Time spent is %s seconds" % self.log.time_since_last_check()
# grammar.print_grammar()
print ">>TREEBANK: Pickling the temporary grammar."
with open("temp_grammar", "w") as file:
pickle.dump(grammar, file)
print ">>LOG: Time spent is %s seconds" % self.log.time_since_last_check()
print ">>TREEBANK: Total runtime %s seconds" % (time.time() - self.log.starttime)
def __init__(self, config): self.connection = Connection(config) self.grammar = Grammar()
elif tag == 'function':
statement = self._function(token, buffer)
else:
raise Exception('BUG: invalid token %s' % tag)
statements.append(statement)
return statements
def define_stmt(self, (tag, left, right, sublist), buffer):
name_tup, value_tup = sublist
value_tag = value_tup[0]
name = getString(name_tup, buffer)
value = getString(value_tup, buffer)
if value_tag == 'regex':
value = self._regex(value_tup, buffer)
elif value_tag == 'varname':
if not self.context.lexicon.has_key(value):
_error(buffer, value_tup[1], 'no such variable')
value = self.context.lexicon[value]
else:
raise Exception('BUG: invalid token %s' % value_tag)
self.context.lexicon[name] = value
def grammar_stmt(self, (tag, left, right, sublist), buffer):
map = singleMap(sublist)
grammar = Grammar()
grammar.name = getString(map['varname'], buffer)
grammar.statements = self._suite(map['suite'], buffer)
if map.has_key('inherit'):
grammar.inherit = self._inherit(map['inherit'], buffer)
self.context.grammars[grammar.name] = grammar
def __init__(self):
self.grammar = Grammar.read_from_file("g1.txt")
self.workingStack = []
self.inputStack = []
self.output = []
def menu():
s = ''
s += '0. Back\n'
s += '1. Set of non-terminals\n'
s += '2. Set of terminals\n'
s += '3. Set of productions\n'
s += '4. Productions for a given non_terminal\n'
s += '5. Parse\n'
print(s)
if __name__ == '__main__':
while True:
grammar_menu()
g = input('>')
grammar = Grammar()
if g == '0':
break
elif g not in ['1', '2']:
continue
grammar.read_file('g' + g + ".txt")
while True:
menu()
i = input('> ')
if i == '1':
print(grammar.non_terminals)
elif i == '2':