def _addToTable(self, group, symbol, item):
row = self.lrtable[group]
if symbol in row:
# there is conflict in lr table
if type(row[symbol]) == TableItem and type(item) == Item:
# probably shift - reduce conflict - no problem now
if not row[symbol].addItem(item):
conflict = "Reduce-reduce"\
if row[symbol].isReduceReduce\
else "Shift-reduce"
debug_print('table', self)
raise ValueError(
conflict + " conflict in lr table on " + "position [" +
str(group) + ", " + str(symbol) + "]", 4)
elif row[symbol] != item:
# two numbers can't be on same place in table
raise ValueError(
"Conflict in lr table:\n[" + str(group) + ", " +
str(symbol) + "] " + "can be " + str(row[symbol]) +
" or " + str(item), 4)
else:
# no conflict
if type(item) == Item:
# create table item object
row[symbol] = TableItem(item)
else:
# simple number
row[symbol] = item
def passing_down_skill(self, skill, villager_list):
debug_print("in passing skill" + skill.skill_name)
cur_leader = self.find_leader(villager_list)
if cur_leader:
debug_print("leader!")
if self.last_skill == None:
skill.applied = True
self.last_skill = skill
self.send_command_to_leader(skill, cur_leader)
return True
else:
debug_print("we've got last skill!")
for one_skill_from_villager in cur_leader.skills:
if self.last_skill.skill_name == one_skill_from_villager.skill_name:
if one_skill_from_villager.applied == False:
debug_print("last skill is not applied")
return False
else:
skill.applied = True
self.last_skill = skill
self.send_command_to_leader(skill, cur_leader)
debug_print("True returned!")
return True
return False
def timeout():
global device_thread
global job_counter
debug_print("No ping for %d seconds, assume all jobs died, timeout." % instance_conf.keep_alive)
# Upload markers and pause
upload_markers(True)
job_counter = 0
device_thread.pause()
def process_marker_buffer():
global marker_up_buffer
global db_cursor
debug_print("Upping %d markers..." % len(marker_up_buffer))
for marker in marker_up_buffer:
u, m = split_timestamp(marker[0])
db_cursor.execute("""INSERT INTO marker_data (guid, time_unix, time_ms, name, marker_type)
VALUES (%s,%s,%s,%s,%s)""", (guid, u, m, marker[1], marker[2]))
db.commit()
marker_up_buffer = []
def handle(self):
global marker_buffer
global marker_buffer_lock
data = self.request[0].strip().split(';')
try:
if data[0] == 'mark':
with marker_buffer_lock:
marker_buffer += [(time.time(), data[1], int(data[2]))]
upload_markers(False)
except IndexError:
debug_print("Malformed request: %s" % data)
def handle(self):
global device_thread
global guid
global job_counter
if not hasattr(self, 'death_timer'):
debug_print("Starting death timer interval %d seconds" % (instance_conf.keep_alive,))
self.death_timer = threading.Timer(instance_conf.keep_alive, timeout)
self.death_timer.start()
data = self.request.recv(1024).strip().split(';')
try:
if data[0] == 'start':
guid = int(random.getrandbits(32))
job_host = data[1]
job_owner = data[2]
job_id = data[3]
job_process = data[4]
db_cursor.execute("""INSERT INTO job_data (guid, job_started, job_host, job_owner, job_id, job_process)
VALUES (%s,NOW(),%s,%s,%s,%s)""", (guid, job_host, job_owner, job_id, job_process))
for device_name, sensors in instance_conf.devices.iteritems():
for sensor in sensors:
db_cursor.execute("""INSERT INTO conf_data_sensor (guid, device_sensor, voltage, description)
VALUES (%s,%s,%s,%s)""", (guid, device_sensor(device_name, sensor), sensor['voltage'], sensor['description']))
db.commit()
device_thread.unpause()
job_counter += 1
elif data[0] == 'ping':
if self.death_timer is not None:
self.death_timer.cancel()
self.death_timer = threading.Timer(instance_conf.keep_alive, timeout)
self.death_timer.start()
elif data[0] == 'stop':
# Up the rest of the markers and pause
upload_markers(True)
job_counter -= 1
if job_counter <= 0:
device_thread.pause()
self.request.send("Command processed.")
except IndexError:
estr = "Malformed request: %s" % data
self.request.send(estr)
debug_print(estr)
def run(self):
line_buffer = collections.deque(maxlen=10)
self.initializing = True
device_file = self.ser.port
while not self._stop.isSet():
self._notpause.wait()
line = self.ser.readline()
# Read lines continuously; ser.readline() will block until a line is read.
# Wait for first "analogzero=" line
# Once encountered, consider the state to be initialized -- ser.readline() will now be piped into line_buffer FIFO
# Wait for "analogzero=" line
# Once encountered, we know that we have read in one page of data
# Parse that page of data and pipe into data_buffer FIFO
if self.initializing:
if "analogzero" in line:
self.initializing = False
else:
if "analogzero" in line:
# empty buffer and push to device data buffer
with self.data_buffer_lock:
if device_file not in self.data_buffer:
self.data_buffer[device_file] = collections.deque(maxlen=MAX_BUFFER)
try:
line_buffer.pop() # pop off trailing \r\n
except IndexError:
continue
if len(line_buffer) > 0:
channels_data = []
t = time.time()
while len(line_buffer) > 0:
try:
x = line_buffer.pop()
channels_data.insert(0, float(x.split('=')[1]))
except IndexError:
debug_print("Possible malformed data.")
channels_data.insert(0, 0.0)
with self.data_buffer_lock:
self.data_buffer[device_file].append((t, channels_data))
debug_print("Data buffer for %s is size %d." % (device_file, len(self.data_buffer[device_file])))
else:
line_buffer.append(line)
def cleanup():
debug_print("Closing devices and database...")
device_handle.close()
db_cursor.close()
instance_conf.read_config(sys.argv[1])
# Open database
db = MySQLdb.connect(host=instance_conf.db_host, port=instance_conf.db_port, db=instance_conf.db_database, user=instance_conf.db_username, passwd=instance_conf.db_password)
db_cursor = db.cursor()
# Open device
for k, v in instance_conf.devices:
device_arr[k] = (serial.Serial(k), ReadDeviceThread())
device_arr[k][1].set_device_handle(device_handle)
device_arr[k][1].set_data_buffer(data_buffer)
device_arr[k][1].set_data_buffer_lock(data_buffer_lock)
device_arr[k][1].start()
debug_print("Device running in thread: %s" % device_arr[k][1].getName())
# Open consumer
consumer = BufferConsumer(consume_period=3, data_buffer=data_buffer, data_buffer_lock=data_buffer_lock, db_connection=db, instance_config=instance_conf)
consumer.start()
debug_print("Consumer running in thread: %s" % consumer.getName())
# Start TCP service
tcp_server = ThreadedTCPServer((instance_conf.host, instance_conf.port), ThreadedTCPRequestHandler)
tcp_server_thread = threading.Thread(target=tcp_server.serve_forever)
tcp_server_thread.setDaemon(True)
tcp_server_thread.start()
debug_print("TCP server running in thread: %s" % tcp_server_thread.getName())
def main():
"""Main function."""
# arguments parsing
argp = ArgumentParser(description='Tree controlled grammar parser',
formatter_class=argparse.RawTextHelpFormatter)
argp.add_argument('-g',
'--grammar',
action='store',
required=True,
type=argparse.FileType('r'),
help='Input grammar file, syntax details in readme.md')
argp.add_argument('-p',
'--print',
default=False,
nargs='+',
action='store',
metavar='CHOICE',
choices=[
'tree', 'trees', 'stack', 'rules', 'groups', 'table',
'eff', 'automat', 'precedence', 'grammar', 'scanner',
'all'
],
help="Decide what to print from these CHOICES:\n" +
" - tree: final derivation tree\n" +
" - trees: derivation tree development\n" +
" - stack: continuously print stack of symbols\n" +
" - rules: continuously print applied rules\n" +
" - groups: lr groups generated from rules\n" +
" - table: lr table\n" +
" - eff: empty, first and follow sets\n" +
" - automat: print final state machine\n" +
" - precedence: print precedence table\n" +
" - grammar: print input grammar\n" +
" - scanner: print input scanner automat\n" +
" - all: print all\n")
argp.add_argument('-i',
'--input',
default=sys.stdin,
action='store',
type=argparse.FileType('r'),
help='Input string file, <stdin> if not present')
argp.add_argument('-o',
'--output',
default=sys.stdout,
action='store',
type=argparse.FileType('w'),
help='Output file, <stdout> if not present')
argp.add_argument('-u')
args = argp.parse_args()
# setup debug_print function to correspond with args
if args.print:
for category in args.print:
Debug.addCategory(category)
if category == 'trees':
Debug.addCategory('tree')
if category == 'all':
Debug.setDebugMode(True)
opened_files = [args.input, args.output, args.grammar]
# set output to output file
sys.stdout = args.output
# parse input grammar
grammarParser = Parser()
try:
grammarParser.parse(args.grammar.read())
except ValueError as e:
# there is an syntax error in input grammar
# print with line number and filename
lineNum = grammarParser.getLine()
charPos = grammarParser.getPos()
closeFiles(opened_files)
err_print(
e.args[1], e.args[0] + "\n(file: '" + args.grammar.name +
"', line: " + str(lineNum) + ", pos: " + str(charPos) + ")")
# get grammar, automat and precednece table from parser
grammar = grammarParser.getGrammar()
debug_print('grammar', grammar, '\n')
automat = grammarParser.getAutomat()
precedence = grammarParser.getPrecedence()
if precedence is not False:
debug_print('precedence', precedence)
# Prepare automat
if automat:
# add new state for lowest level of the tree
# there can be all terminals
startState = automat.getStart()
newState = "t*"
automat.addState(newState)
automat.setTerminating(newState)
for term in grammar.terminals:
automat.addRule(startState, term, newState)
automat.addRule(newState, term, newState)
# determinate automat
automat.dropERules()
automat.determinate()
debug_print('automat', automat)
# lltable = ll_table.LLTable(grammar)
# print(lltable)
# exit(0)
try:
# build lr table
table = LRTable(grammar, precedence, automat)
except ValueError as e:
# error in grammar
closeFiles(opened_files)
err_print(e.args[1], e.args[0])
# parse input string
scanner = InputParser(args.input.read(), grammar.terminals)
debug_print('scanner', scanner.aut, '\n')
# analyze input symbols
try:
tree = table.analyzeSymbols(scanner.getToken)
debug_print('tree', tree, '\n')
except ValueError as e:
# error in input string
lineNum = scanner.getLine()
charPos = scanner.getPos()
closeFiles(opened_files)
err_print(
e.args[1], e.args[0] + "\n(file: '" + args.input.name +
"', line: " + str(lineNum) + ", pos: " + str(charPos) + ")")
closeFiles(opened_files)
sys.exit(0)
def _getItem(self, stack, alpha, state, token, tree):
grammar = self.grammar
item = False
# print(alpha.isShiftReduce, alpha.isReduceReduce)
if alpha.isShiftReduce or alpha.isReduceReduce:
if self.automat is not False:
# try to solve it by tree conflict
# reduce-reduce conflict
possibleRules = []
debug_print('rules', "conflict:")
for it in alpha.getReduce():
rule = grammar.rules[it.state]
states = tree.tryApplyRule(rule)
if states is not False:
debug_print('rules', " ", rule, "\t ok")
possibleRules.append((it, rule, states))
else:
debug_print('rules', " ", rule, "\t no")
if len(possibleRules) > 1:
# multiple options
# we don't know what rule use - it is possible
# that some way leads to success
self.exit_code = 2
options = "\n".join([str(pos[1]) for pos in possibleRules])
raise ValueError("Unhandled reduce-reduce " +
"conflict in lr table on position [" +
str(state) + "," + token + "], " +
"got theese options:\n" + options)
elif len(possibleRules) == 1:
# got only one option
pos = possibleRules[0]
if alpha.isShiftReduce:
debug_print('rules', "undeterministic reduce")
# we are just guessing - if there
# gonna be fail in future that doesn't
# mean, that string can't be in grammar
self.exit_code = 2
else:
debug_print('rules', "deterministic reduce")
tree.applyRule(pos[1], pos[2])
item = pos[0]
else:
if alpha.isShiftReduce:
# if there is conflict, just shift
debug_print('rules', "shift", token)
item = alpha.getItem(Operation.shift)
tree.pushSymbol(token)
else:
raise ValueError("No rule fits to tree for " +
"reduce-reduce conflict in lr " +
"table, on position [" + str(state) +
"," + token + "]")
elif item is False:
# we don't know what to do - shift-reduce conflict
self.exit_code = 2
raise ValueError("Unhandled " +
"conflict in lr table on position [" +
str(state) + "," + token + "], ")
else:
# no conflict, get operation
item = alpha.getItem()
if item.operation == Operation.shift:
tree.pushSymbol(token)
debug_print('rules', "shift", token)
else:
if not tree.applyRule(grammar.rules[item.state]):
raise ValueError("Rule " + str(grammar.rules[item.state]) +
" can't be used, because " +
"of tree conflict.")
else:
debug_print('rules', grammar.rules[item.state])
return item
def analyzeSymbols(self, getToken):
"""Analyze symbols by ll_table."""
grammar = self.grammar
table = self.lrtable
automat = self.automat
# put end symbol and state 0 on stack
stack = [StackItem('', 0)]
state = 0
token = getToken()
tree = Tree(automat, self.grammar)
err = False
self.exit_code = 1
try:
while True:
if Debug.isActivated('stack'):
debug_print('stack', state,
"".join([str(item) for item in stack]))
if token not in self.grammar.terminals and token != '':
# input symbol is not in grammar alphabet
# no matter what this string doesn't bellow to grammar
self.exit_code = 1
raise ValueError("Symbol '" + token +
"' is not in grammar alphabet.")
if token not in table[state]:
# no rule for this symbol and state
raise ValueError("No rule for token '" + token +
"' in state " + str(state))
# get item from table
alpha = table[state][token]
if alpha == -1:
# we are at the end
break
# get item (solve conflicts)
item = self._getItem(stack, alpha, state, token, tree)
if item.operation == Operation.shift:
# shift - add symbol to stack
stack.append(StackItem(token, item.state))
token = getToken()
state = item.state
else:
# reduce - apply rule
rule = grammar.rules[item.state]
for s1 in reversed(rule.rightSide):
# check, that symbols on stack are same with right side
# of the rule
pSymbol = stack.pop()
if pSymbol.symbol != s1:
raise ValueError("Expecting '" + str(s1) +
"', got '" + str(pSymbol) +
"' from rule " + str(rule))
# get state of last symbol on stack
symbolState = stack[-1].state
# get new state from beta table
state = table[symbolState][rule.leftSide]
# add rule left side to stack
stack.append(StackItem(rule.leftSide, state))
debug_print('trees', tree, '\n')
tree.checkTree()
except ValueError as e:
err = ValueError(e.args[0], self.exit_code)
if err:
debug_print('tree', tree)
raise err
return tree
def __init__(self, grammar, precedence, automat):
"""Initialization."""
self.grammar = grammar
self.automat = automat
self.lrtable = []
self.precendece = precedence
# setup table item for allowing conflicts
# based on what instruments (automat, precedence) are available
TableItem.setConflicts(
self.automat is not False or self.precendece is not False,
self.automat is not False)
additionalRule = Rule('S*', [self.grammar.start])
self.grammar.rules = [additionalRule] + self.grammar.rules
self.grammar.nonterminals = ['S*'] + self.grammar.nonterminals
firstRule = LRRule(additionalRule, 0)
# add priorities to rules by their order
for i, rule in enumerate(reversed(self.grammar.rules)):
rule.priority = i
self.groups = LRGroups(grammar, firstRule)
debug_print('groups', self.groups, '\n')
groups = self.groups
self.eff = EFF(grammar)
debug_print('eff', self.eff, '\n')
endRule = LRRule(additionalRule, 1)
lrtable = self.lrtable
for i, group in enumerate(groups.groups):
lrtable.append({})
for rule in group.rules:
if rule == endRule:
# end point marked as -1
self._addToTable(i, '', -1)
continue
markedS = rule.getMarkedSymbol()
if markedS is False:
# marked symbol on last position - reduce operation
follow = self.eff.follow(rule.r.leftSide)
for symbol in follow:
item = Item(self.grammar.rules.index(rule.r),
Operation.reduce)
self._addToTable(i, symbol, item)
elif not self.grammar.isTerm(markedS):
# marked symbol is non-terminal - beta part of table
self._addToTable(i, markedS, group.transitions[markedS])
else:
# marked symbol is terminal - shift operation
tIt = Item(group.transitions[markedS], Operation.shift)
self._addToTable(i, markedS, tIt)
if self.precendece:
for i, row in enumerate(lrtable):
for symbol in row:
item = row[symbol]
if isinstance(item, TableItem) and item.isShiftReduce and\
not item.isReduceReduce:
rule = self.grammar.rules[item.getItem(
Operation.reduce).state]
op = self.precendece.getPrecedence(
rule.rightSide, symbol)
# print(rule, symbol, op)
if op:
item.setOperation(op)
elif self.automat is False:
debug_print('table', self)
raise ValueError(
"Conflict in LR table " + "on position [" +
str(i) + ", " + str(symbol) +
"] can't be handled be " + "precendece table",
4)
debug_print('table', self)
