class PulseProgram: """ Encapsulates a PulseProgrammer Program loadSource( filename ) loads the contents of the file The code is compiled in the following steps parse() generates self.code toBytecode() generates self.bytecode toBinary() generates self.binarycode the procedure updateVariables( dictionary ) updates variable values in the bytecode """ def __init__(self): self.variabledict = collections.OrderedDict( ) # keeps information on all variables to easily change them later self.labeldict = dict() # keep information on all labels self.source = collections.OrderedDict( ) # dictionary of source code files (stored as strings) self.code = [] # this is a list of lines self.bytecode = [] # list of op, argument tuples self.binarycode = bytearray() # binarycode to be uploaded self._exitcodes = dict( ) # generate a reverse dictionary of variables of type exitcode self.constDict = dict() class Board: channelLimit = 1 halfClockLimit = 500000000 self.adIndexList = [(x, 0) for x in range(6)] self.adBoards = [Board()] * 6 self.timestep = Q(20.0, 'ns') self.initBNF() def initBNF(self): constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action) vardecl = (VAR + NAME + VALUE + Optional(COMMA + Regex("[^#\n]*"))).setParseAction( self.var_action) insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action) LABEL = IDENTIFIER + COLON COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress()) COMMAND = COMMANDEXP.setParseAction(self.command_action) LABELEDCOMMAND = (LABEL + COMMANDEXP).setParseAction( self.label_command_action) decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def const_action(self, text, loc, arg): """ add the const to the self.constDict dictionary """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg)) label, value = arg if label in self.constDict: logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'" .format(self.currentFile, label, value, self.constDict[label])) raise ppexception("Redefining variable", self.currentFile, lineno, label) else: self.constDict[label] = int(value) def var_action(self, text, loc, arg): print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg) > 2 else "") """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} Variable {2}".format(self.currentFile, lineno(loc, text), arg)) var = Variable() label, data = arg[:2] fields = arg[2].split(",") if len(arg) > 2 else [None] * 3 fields += [None] * (3 - len(fields)) var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ] var.name = label var.origin = self.currentFile var.enabled = True if var.encoding not in encodings: raise ppexception( "unknown encoding {0} in file '{1}':{2}".format( var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format( self.currentFile, data)) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter(var.value, var.encoding) else: var.value = Q(float(data)) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." % (self.currentFile, label, data, self.defines[label])) raise ppexception("variable redifinition", self.currentFile, lineno, label) else: self.defines[ label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var def command_action(self, text, loc, arg): print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg) > 1 else "") def label_command_action(self, text, loc, arg): print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg) > 2 else "") def addLabel(self, label, address, sourcename, lineno): if label is not None: self.labeldict[label] = address def insert_action(self, text, loc, arg): oldfile = self.currentFile print("insert_action", lineno(loc, text), arg) myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile(self.currentFile) self.currentFile = oldfile print(result) return result def assembleFile(self, filename): self.currentFile = filename result = self.program.parseFile(self.currentFile) return result def setHardware(self, adIndexList, adBoards, timestep): self.adIndexList = adIndexList self.adBoards = adBoards self.timestep = timestep assert self.timestep.has_dimension('s') def saveSource(self): for name, text in self.source.items(): with open(os.path.join(self.pp_dir, name), 'w') as f: f.write(text) def loadSource(self, pp_file): """ Load the source pp_file #include files are loaded recursively all code lines are added to self.sourcelines for each source file the contents are added to the dictionary self.source """ self.source.clear() self.pp_dir, self.pp_filename = os.path.split(pp_file) self.sourcelines = [] self.insertSource(self.pp_filename) self.compileCode() def updateVariables(self, variables): """ update the variable values in the bytecode """ logger = logging.getLogger(__name__) for name, value in variables.items(): if name in self.variabledict: var = self.variabledict[name] address = var.address var.value = value logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}" .format(name, value, address, int(var.data))) var.data = self.convertParameter(value, var.encoding) self.bytecode[address] = (self.bytecode[address][0], var.data) self.variabledict[name] = var else: logger.error( "variable {0} not found in dictionary.".format(name)) return self.bytecode def variables(self): mydict = dict() for name, var in self.variabledict.items(): mydict.update({name: var.value}) return mydict def variable(self, variablename): return self.variabledict.get(variablename).value def variableUpdateCode(self, variablename, value): """returns the code to update the variable directly on the fpga consists of variablelocation and variablevalue """ var = self.variabledict[variablename] data = self.convertParameter(value, var.encoding) return bytearray(struct.pack('II', (var.address, data))) def flattenList(self, l): return [item for sublist in l for item in sublist] def variableScanCode(self, variablename, values): var = self.variabledict[variablename] # [item for sublist in l for item in sublist] idiom for flattening of list return self.flattenList([(var.address, self.convertParameter(x, var.encoding)) for x in values]) def loadFromMemory(self): """Similar to loadSource only this routine loads from self.source """ self.sourcelines = [] self._exitcodes = dict() self.insertSource(self.pp_filename) self.compileCode() def toBinary(self): """ convert bytecode to binary """ logger = logging.getLogger(__name__) self.binarycode = bytearray() for wordno, (op, arg) in enumerate(self.bytecode): logger.debug("{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op) << 24) + int(arg))))) self.binarycode += struct.pack('I', int((op << 24) + arg)) return self.binarycode def currentVariablesText(self): lines = list() for name, var in iter(sorted(self.variabledict.items())): lines.append("{0} {1}".format(name, var.value)) return '\n'.join(lines) # routines below here should not be needed by the user insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE) codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE) def insertSource(self, pp_file): """ read a source file pp_file calls itself recursively to for #insert adds the contents of this file to the dictionary self.source """ logger = logging.getLogger(__name__) if pp_file not in self.source: with open(os.path.join(self.pp_dir, pp_file)) as f: self.source[pp_file] = ''.join(f.readlines()) sourcecode = self.source[pp_file] for line, text in enumerate(sourcecode.splitlines()): m = self.insertPattern.match(text) if m: filename = m.group(1) logger.info("inserting code from {0}".format(filename)) self.insertSource(filename) else: if self.codelinePattern.match(text): self.sourcelines.append((text, line + 1, pp_file)) labelPattern = re.compile('(\w+):\s+([^#\n\r]*)') opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE) varPattern = re.compile( 'var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}' ) # def parse(self): """ parse the code """ logger = logging.getLogger(__name__) self.code = [] self.variabledict = collections.OrderedDict() self.defines = dict() addr_offset = 0 for text, lineno, sourcename in self.sourcelines: m = self.varPattern.match(text) if m: self.addVariable(m, lineno, sourcename) else: m = self.definePattern.match(text) if m: self.addDefine(m, lineno, sourcename) else: # extract any JMP label, if present m = self.labelPattern.match(text) if m: label, text = m.groups( ) #so the operation after ":" still gets parsed CWC 08162012 else: label = None #The label for non-jump label line is NONE CWC 08172012 # search OPS list for a match to the current line m = self.opPattern.match(text) if m: op, args = m.groups() op = op.upper() # split and remove whitespace arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',') ] #substitute the defined variable directly with the corresponding value CWC 08172012 arglist = [ self.defines[x] if x in self.defines else x for x in arglist ] #check for dds commands so CHAN commands can be inserted if (op[:3] == 'DDS'): try: board = self.adIndexList[int(arglist[0])][0] except ValueError: raise ppexception( "DDS argument does not resolve to integer", sourcename, lineno, arglist[0]) chan = self.adIndexList[int(arglist[0])][1] if (self.adBoards[board].channelLimit != 1): #boards with more than one channel require an extra channel selection command chanData = self.adBoards[board].addCMD(chan) chanData += (int(board) << 16) self.code.append( (len(self.code) + addr_offset, 'DDSCHN', chanData, label, sourcename, lineno)) data = arglist if len(arglist) > 1 else arglist[0] self.addLabel(label, len(self.code), sourcename, lineno) self.code.append((len(self.code) + addr_offset, op, data, label, sourcename, lineno)) else: logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)" .format(text, sourcename, lineno)) raise ppexception( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)" .format(text, sourcename, lineno), sourcename, lineno, text) self.appendVariableCode() return self.code def appendVariableCode(self): """ append all variables to the instruction part of the code """ for var in list(self.variabledict.values()): address = len(self.code) self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0)) var.address = address def addVariable(self, m, lineno, sourcename): """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug("Variable {0} {1} {2}".format(m.groups(), lineno, sourcename)) var = Variable() label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups() ] var.name = label var.origin = sourcename var.enabled = True if var.encoding not in encodings: raise ppexception( "unknown encoding {0} in file '{1}':{2}".format( var.encoding, sourcename, lineno), sourcename, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format( sourcename, data)) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter(var.value, var.encoding) else: var.value = Q(float(data)) # var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." % (sourcename, label, data, self.defines[label])) raise ppexception("variable redifinition", sourcename, lineno, label) else: self.defines[ label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var # code is (address, operation, data, label or variablename, currentfile) def toBytecode(self): """ generate bytecode from code """ logger = logging.getLogger(__name__) logger.debug("\nCode ---> ByteCode:") self.bytecode = [] for line in self.code: logger.debug("{0}: {1}".format(hex(line[0]), line[1:])) bytedata = 0 if line[1] not in OPS: raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1]) byteop = OPS[line[1]] try: data = line[2] #attempt to locate commands with constant data if (data == ''): #found empty data bytedata = 0 elif isinstance(data, (int, int)): bytedata = data elif isinstance(data, float): bytedata = int(data) elif isinstance( data, str ): # now we are dealing with a variable and need its address bytedata = self.variabledict[line[2]].address if line[ 2] in self.variabledict else self.labeldict[line[2]] elif isinstance( data, list ): # list is what we have for DDS, will have 8bit channel and 16bit address channel, data = line[2] if isinstance(data, str): data = self.variabledict[data].address bytedata = ( (int(channel) & 0xf) << 16) | (int(data) & 0x0fff) except KeyError: logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n" .format(line[4], data)) raise ppexception( "{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data) self.bytecode.append((byteop, bytedata)) logger.debug("---> {0} {1}".format(hex(byteop), hex(bytedata))) return self.bytecode def convertParameter(self, mag, encoding=None): """ convert a dimensioned parameter to the binary value expected by the hardware. The conversion is determined by the variable encoding """ if is_Q(mag): if mag.dimensionality == Dimensions.time: result = int(round(mag / self.timestep)) else: step, unit, _, mask = encodings[encoding] result = int(round(mag.m_as(unit) / step)) & mask else: if encoding: step, unit, _, mask = encodings[encoding] result = int(round(mag / step)) & mask else: result = mag return result def compileCode(self): self.parse() self.toBytecode() def exitcode(self, code): if code in self._exitcodes: var = self._exitcodes[code] if var.comment: return var.comment else: return var.name else: return "Exitcode {0} Not found".format(code)
class pppCompiler: def __init__(self): self.initBNF() self.symbols = SymbolTable() def initBNF(self): indentStack = [1] encoding = Literal("<").suppress() + identifier("encoding") + Literal( ">").suppress() constdecl = Group((const + identifier + assign + value).setParseAction( self.const_action)) vardecl = Group( (type_("type_") + Optional(encoding) + identifier("name") + Optional(assign + value("value") + Optional(identifier)("unit")) ).setParseAction(self.var_action)) insertdecl = Group( (insert + dblQuotedString + LineEnd().suppress()).setParseAction( self.insert_action)) procedurecall = Group((identifier + Literal("(").suppress() + Optional( delimitedList( (identifier + Optional(assign + identifier)).setParseAction( self.named_param_action))) + Literal(")").suppress()).setParseAction( self.procedurecall_action)) condition = Group( (identifier("leftidentifier") + comparison("comparison") + (identifier("identifier") | value.setParseAction(self.value_action))).setParseAction( self.condition_action))("condition") pointer = Literal("*") + identifier rExp = Forward() #numexpression = Forward() opexpression = (identifier("operand") + (Literal(">>") | Literal("<<") | Literal("+") | Literal("*") | Literal("/") | Literal("-"))("op") + Group(rExp)("argument")).setParseAction( self.opexpression_action) rExp << ( procedurecall | opexpression | identifier("identifier") | value.setParseAction(self.value_action) | #Group( Suppress("(") + rExp + Suppress(")") ) | #Group( "+" + rExp) | #Group( "-" + rExp) | Group(Literal("not") + rExp)) rExpCondition = Group( (Optional(not_)("not_") + rExp("rExp"))).setParseAction( self.rExp_condition_action)("condition") rExp.setParseAction(self.rExp_action) assignment = ((identifier | pointer)("lval") + assign + rExp("rval")).setParseAction(self.assignment_action) addassignment = ((identifier | pointer)("lval") + (Literal("+=") | Literal("-=") | Literal("*=") | Literal("&=") | Literal("|=") | Literal(">>=") | Literal("/=") | Literal("<<="))("op") + Group(rExp)("rval")).setParseAction( self.addassignement_action) statement = Forward() statementBlock = indentedBlock(statement, indentStack).setParseAction( self.statementBlock_action) procedure_statement = Group( (Keyword("def").suppress() + identifier("funcname") + Literal("(").suppress() + Literal(")").suppress() + colon.suppress() + statementBlock).setParseAction( self.def_action)) while_statement = Group( (Keyword("while").suppress() + (condition | rExpCondition)("condition") + colon.suppress() + statementBlock("statementBlock")).setParseAction( self.while_action)) if_statement = (Keyword("if") + condition + colon + statementBlock("ifblock") + Optional( Keyword("else").suppress() + colon + statementBlock("elseblock"))).setParseAction( self.if_action) statement << (procedure_statement | while_statement | if_statement | procedurecall | assignment | addassignment) decl = constdecl | vardecl | insertdecl | Group(statement) self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def assignment_action(self, text, loc, arg): logging.getLogger(__name__).debug("assignment_action {0} {1}".format( lineno(loc, text), arg)) try: code = [ "# line {0} assignment {1}".format(lineno(loc, text), line(loc, text)) ] rval_code = find_and_get(arg.rval, 'code') if rval_code is not None: code += arg.rval.code elif arg.rval == "*P": code.append(" LDWI") elif 'identifier' in arg: self.symbols.getVar(arg.identifier) code.append(" LDWR {0}".format(arg.identifier)) if arg.lval == "*P": code.append(" STWI") elif arg.lval != "W": symbol = self.symbols.getVar(arg.lval) code.append(" STWR {0}".format(symbol.name)) if 'code' in arg: arg['code'].extend(code) else: arg['code'] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def addassignement_action(self, text, loc, arg): logging.getLogger(__name__).debug( "addassignement_action {0} {1}".format(lineno(loc, text), arg)) try: code = [ "# line {0}: add_assignment: {1}".format( lineno(loc, text), line(loc, text)) ] if arg.rval[0] == '1' and arg.op in ['+=', '-=']: self.symbols.getVar(arg.lval) if arg.op == "+=": code.append(" INC {0}".format(arg.lval)) else: code.append(" DEC {0}".format(arg.lval)) else: if 'code' in arg.rval: code += arg.rval.code self.symbols.getVar(arg.lval) if arg.op == "-=": raise CompileException( "-= with expression needs to be fixed in the compiler" ) code.append(" {0} {1}".format(opassignmentLookup[arg.op], arg.lval)) elif 'identifier' in arg.rval: self.symbols.getVar(arg.rval.identifier) code.append(" LDWR {0}".format(arg.lval)) self.symbols.getVar(arg.lval) code.append(" {0} {1}".format(opassignmentLookup[arg.op], arg.rval.identifier)) code.append(" STWR {0}".format(arg.lval)) arg['code'] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def condition_action(self, text, loc, arg): logging.getLogger(__name__).debug("condition_action {0} {1}".format( lineno(loc, text), arg)) try: code = [ "# line {0} condition {1}".format(lineno(loc, text), line(loc, text)) ] if arg.leftidentifier != "W": self.symbols.getVar(arg.leftidentifier) code.append(' LDWR {0}'.format(arg.leftidentifier)) if arg.identifier == 'NULL' and arg.comparison in jmpNullCommands: arg['jmpcmd'] = jmpNullCommands[arg.comparison] else: code.append(' {0} {1}'.format( comparisonCommands[arg.comparison], arg.identifier)) arg["code"] = code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def rExp_condition_action(self, text, loc, arg): logging.getLogger(__name__).debug( "rExp_condition_action {0} {1}".format(lineno(loc, text), arg)) try: code = [ "# line {0} rExp_condition {1}".format(lineno(loc, text), line(loc, text)) ] condition_code = arg.condition.rExp['code'] if isinstance(condition_code, str): if 'not_' in arg['condition']: code += [" CMPEQUAL NULL"] else: code += [" CMPNOTEQUAL NULL"] arg['code'] = code else: if 'not_' in arg['condition']: arg['code'] = { False: condition_code[True], True: condition_code[False] } else: arg['code'] = condition_code except Exception as e: raise CompileException(text, loc, str(e), self) return arg def named_param_action(self, text, loc, arg): if len(arg) == 2: arg[arg[0]] = arg[1] return arg def value_action(self, text, loc, arg): if arg[0][0:2] == '0x': value = int(arg[0], 16) else: value = int(arg[0]) arg["identifier"] = self.symbols.getInlineParameter("inlinevar", value) return arg def opexpression_action(self, text, loc, arg): try: logging.getLogger(__name__).debug( "opexpression_action {0} {1}".format(lineno(loc, text), arg)) code = [ "# line {0}: shiftexpression {1}".format( lineno(loc, text), line(loc, text)), " LDWR {0}".format(arg.operand), " {0} {1}".format(shiftLookup[arg.op], arg.argument.identifier) ] arg['code'] = code logging.getLogger(__name__).debug( "shiftexpression generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def procedurecall_action(self, text, loc, arg): try: logging.getLogger(__name__).debug( "procedurecall_action {0} {1}".format(lineno(loc, text), arg)) procedure = self.symbols.getProcedure(arg[0]) code = [ "# line {0}: procedurecall {1}".format(lineno(loc, text), line(loc, text)) ] opcode = procedure.codegen(self.symbols, arg=arg.asList(), kwarg=arg.asDict()) if isinstance(opcode, list): code += opcode else: code = opcode arg['code'] = code logging.getLogger(__name__).debug( "procedurecall generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def rExp_action(self, text, loc, arg): logging.getLogger(__name__).debug("rExp_action {0} {1}".format( lineno(loc, text), arg)) pass def if_action(self, text, loc, arg): logging.getLogger(__name__).debug("if_action {0} {1}".format( lineno(loc, text), arg)) try: block0 = [ "# line {0} if statement {1}".format(lineno(loc, text), line(loc, text)) ] if isinstance(arg.condition.code, list): block0 += arg.condition.code JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False] else: JMPCMD = arg.condition.code[True] if 'elseblock' in arg: block1 = arg.ifblock.ifblock.code block2 = arg.elseblock.elseblock[ 'code'] if 'elseblock' in arg.elseblock else arg.elseblock[ 'code'] else: block1 = arg.ifblock.ifblock['code'] block2 = None arg['code'] = [ IfGenerator(self.symbols, JMPCMD, block0, block1, block2) ] except Exception as e: raise CompileException(text, loc, str(e), self) return arg def while_action(self, text, loc, arg): logging.getLogger(__name__).debug("while_action {0} {1}".format( lineno(loc, text), arg)) try: block0 = [ "# line {0} while_statement {1}".format( lineno(loc, text), line(loc, text)) ] if 'code' in arg.condition: if isinstance(arg.condition.code, list): block1 = arg.condition.code JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False] else: JMPCMD = arg.condition.code[True] block1 = [] elif 'rExp' in arg.condition and 'code' in arg.condition.rExp: if isinstance(arg.condition.rExp.code, list): block1 = arg.condition.rExp.code JMPCMD = arg.condition.rExp.get('jmpcmd', "JMPNCMP") else: JMPCMD = arg.condition.rExp.code[True] block1 = [] block2 = arg.statementBlock.statementBlock['code'] arg['code'] = [ WhileGenerator(self.symbols, JMPCMD, block0, block1, block2) ] logging.getLogger(__name__).debug("while_action generated code ") except Exception as e: raise CompileException(text, loc, str(e), self) return arg def statementBlock_action(self, text, loc, arg): logging.getLogger(__name__).debug( "statementBlock_action {0} {1} {2}".format(lineno(loc, text), arg.funcname, arg)) try: code = list() for command in arg[0]: if 'code' in command: code += command['code'] elif 'code' in command[0]: code += command[0]['code'] arg[0]['code'] = code logging.getLogger(__name__).debug( "statementBlock generated code {0}".format(code)) except Exception as e: raise CompileException(text, loc, str(e), self) return arg def def_action(self, text, loc, arg): logging.getLogger(__name__).debug("def_action {0} {1} {2}".format( lineno(loc, text), arg.funcname, arg)) try: name = arg[0] self.symbols.checkAvailable(name) self.symbols[name] = FunctionSymbol(name, arg[1]['code']) except Exception as e: raise CompileException(text, loc, str(e), self) def const_action(self, text, loc, arg): try: name, value = arg logging.getLogger(__name__).debug( "const_action {0} {1} {2} {3}".format(self.currentFile, lineno(loc, text), name, value)) self.symbols[name] = ConstSymbol(name, value) except Exception as e: raise CompileException(text, loc, str(e), self) def var_action(self, text, loc, arg): logging.getLogger(__name__).debug( "var_action {0} {1} {2} {3} {4} {5} {6}".format( self.currentFile, lineno(loc, text), arg["type_"], arg.get("encoding"), arg["name"], arg.get("value"), arg.get("unit"))) try: type_ = arg["type_"] if arg["type_"] != "var" else None self.symbols[arg["name"]] = VarSymbol(type_=type_, name=arg["name"], value=arg.get("value"), encoding=arg.get("encoding"), unit=arg.get("unit")) except Exception as e: raise CompileException(text, loc, str(e), self) def insert_action(self, text, loc, arg): try: oldfile = self.currentFile myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile(self.currentFile) self.currentFile = oldfile except Exception as e: raise CompileException(text, loc, str(e), self) return result def compileFile(self, filename): self.currentFile = filename result = self.program.parseFile(self.currentFile, parseAll=True) allcode = list() for element in result: if not isinstance(element, str) and 'code' in element: allcode += element['code'] elif not isinstance(element[0], str) and 'code' in element[0]: allcode += element[0]['code'] header = self.createHeader() codetext = "\n".join(header + allcode) return codetext def compileString(self, programText): self.programText = programText self.currentFile = "Memory" result = self.program.parseString(self.programText, parseAll=True) allcode = list() for element in result: if not isinstance(element, str) and 'code' in element: allcode += element['code'] elif not isinstance(element[0], str) and 'code' in element[0]: allcode += element[0]['code'] header = self.createHeader() codetext = """# autogenerated # DO NOT EDIT DIRECTLY # The file will be overwritten by the compiler # """ codetext += "\n".join(header + list(generate(allcode))) self.reverseLineLookup = self.generateReverseLineLookup(codetext) return codetext def generateReverseLineLookup(self, codetext): lookup = dict() sourceline = None for codeline, line in enumerate(codetext.splitlines()): m = re.search('^\# line (\d+).*$', line) if m: sourceline = int(m.group(1)) else: lookup[codeline + 1] = sourceline return lookup def createHeader(self): header = ["# const values"] for constval in self.symbols.getAllConst(): header.append("const {0} {1}".format(constval.name, constval.value)) header.append("# variables ") for var in self.symbols.getAllVar(): if var.type_ == "masked_shutter": header.append("var {0} {1}, {2}".format( var.name + "_mask", var.value if var.value is not None else 0, "mask")) header.append("var {0} {1}, {2}".format( var.name, var.value if var.value is not None else 0, "shutter {0}_mask".format(var.name))) else: optionals = [ s if s is not None else "" for s in list_rtrim([var.type_, var.unit, var.encoding]) ] varline = "var {0} {1}".format( var.name, var.value if var.value is not None else 0) if len(optionals) > 0: varline += ", " + ", ".join(optionals) header.append(varline) header.append("# inline variables") # for value, name in self.symbols.inlineParameterValues.items(): # header.append("var {0} {1}".format(name, value)) header.append("# end header") header.append("") return header
class PulseProgram: """ Encapsulates a PulseProgrammer Program loadSource( filename ) loads the contents of the file The code is compiled in the following steps parse() generates self.code toBytecode() generates self.bytecode toBinary() generates self.binarycode the procedure updateVariables( dictionary ) updates variable values in the bytecode """ def __init__(self): self.variabledict = collections.OrderedDict() # keeps information on all variables to easily change them later self.labeldict = dict() # keep information on all labels self.source = collections.OrderedDict() # dictionary of source code files (stored as strings) self.code = [] # this is a list of lines self.bytecode = [] # list of op, argument tuples self.binarycode = bytearray() # binarycode to be uploaded self._exitcodes = dict() # generate a reverse dictionary of variables of type exitcode self.constDict = dict() class Board: channelLimit = 1 halfClockLimit = 500000000 self.adIndexList = [(x, 0) for x in range(6) ] self.adBoards = [ Board() ]*6 self.timestep = Q(20.0, 'ns') self.initBNF() def initBNF(self): constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action) vardecl = (VAR + NAME + VALUE + Optional( COMMA + Regex("[^#\n]*")) ).setParseAction(self.var_action) insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action) LABEL = IDENTIFIER + COLON COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress() ) COMMAND = COMMANDEXP.setParseAction(self.command_action) LABELEDCOMMAND = (LABEL + COMMANDEXP ).setParseAction(self.label_command_action) decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND self.program = ZeroOrMore(decl) self.program.ignore(pythonStyleComment) def const_action( self, text, loc, arg ): """ add the const to the self.constDict dictionary """ logger = logging.getLogger(__name__) logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg)) label, value = arg if label in self.constDict: logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'".format(self.currentFile, label, value, self.constDict[label]) ) raise ppexception("Redefining variable", self.currentFile, lineno, label) else: self.constDict[label] = int(value) def var_action( self, text, loc, arg): print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "") """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug( "{0}:{1} Variable {2}".format( self.currentFile, lineno(loc, text), arg ) ) var = Variable() label, data = arg[:2] fields = arg[2].split(",") if len(arg)>2 else [None]*3 fields += [None]*(3-len(fields)) var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ] var.name = label var.origin = self.currentFile var.enabled = True if var.encoding not in encodings: raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(self.currentFile, data) ) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter( var.value, var.encoding ) else: var.value = Q(float(data)) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(self.currentFile, label, data, self.defines[label]) ) raise ppexception("variable redifinition", self.currentFile, lineno, label) else: self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({ label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var def command_action( self, text, loc, arg): print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg)>1 else "") def label_command_action( self, text, loc, arg): print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "") def addLabel(self, label, address, sourcename, lineno): if label is not None: self.labeldict[label] = address def insert_action( self, text, loc, arg ): oldfile = self.currentFile print("insert_action", lineno(loc, text), arg) myprogram = self.program.copy() self.currentFile = arg[0][1:-1] result = myprogram.parseFile( self.currentFile ) self.currentFile = oldfile print(result) return result def assembleFile(self, filename): self.currentFile = filename result = self.program.parseFile( self.currentFile ) return result def setHardware(self, adIndexList, adBoards, timestep ): self.adIndexList = adIndexList self.adBoards = adBoards self.timestep = timestep assert self.timestep.has_dimension('s') def saveSource(self): for name, text in self.source.items(): with open(os.path.join(self.pp_dir, name), 'w') as f: f.write(text) def loadSource(self, pp_file): """ Load the source pp_file #include files are loaded recursively all code lines are added to self.sourcelines for each source file the contents are added to the dictionary self.source """ self.source.clear() self.pp_dir, self.pp_filename = os.path.split(pp_file) self.sourcelines = [] self.insertSource(self.pp_filename) self.compileCode() def updateVariables(self, variables ): """ update the variable values in the bytecode """ logger = logging.getLogger(__name__) for name, value in variables.items(): if name in self.variabledict: var = self.variabledict[name] address = var.address var.value = value logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}".format(name, value, address, int(var.data)) ) var.data = self.convertParameter(value, var.encoding ) self.bytecode[address] = (self.bytecode[address][0], var.data ) self.variabledict[name] = var else: logger.error( "variable {0} not found in dictionary.".format(name) ) return self.bytecode def variables(self): mydict = dict() for name, var in self.variabledict.items(): mydict.update( {name: var.value }) return mydict def variable(self, variablename ): return self.variabledict.get(variablename).value def variableUpdateCode(self, variablename, value ): """returns the code to update the variable directly on the fpga consists of variablelocation and variablevalue """ var = self.variabledict[variablename] data = self.convertParameter(value, var.encoding ) return bytearray( struct.pack('II', (var.address, data))) def flattenList(self, l): return [item for sublist in l for item in sublist] def variableScanCode(self, variablename, values): var = self.variabledict[variablename] # [item for sublist in l for item in sublist] idiom for flattening of list return self.flattenList( [ (var.address, self.convertParameter(x, var.encoding)) for x in values ] ) def loadFromMemory(self): """Similar to loadSource only this routine loads from self.source """ self.sourcelines = [] self._exitcodes = dict() self.insertSource(self.pp_filename) self.compileCode() def toBinary(self): """ convert bytecode to binary """ logger = logging.getLogger(__name__) self.binarycode = bytearray() for wordno, (op, arg) in enumerate(self.bytecode): logger.debug( "{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op)<<24) + int(arg))) ) ) self.binarycode += struct.pack('I', int((op<<24) + arg)) return self.binarycode def currentVariablesText(self): lines = list() for name, var in iter(sorted(self.variabledict.items())): lines.append("{0} {1}".format(name, var.value)) return '\n'.join(lines) # routines below here should not be needed by the user insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE) codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE) def insertSource(self, pp_file): """ read a source file pp_file calls itself recursively to for #insert adds the contents of this file to the dictionary self.source """ logger = logging.getLogger(__name__) if pp_file not in self.source: with open(os.path.join(self.pp_dir, pp_file)) as f: self.source[pp_file] = ''.join(f.readlines()) sourcecode = self.source[pp_file] for line, text in enumerate(sourcecode.splitlines()): m = self.insertPattern.match(text) if m: filename = m.group(1) logger.info( "inserting code from {0}".format(filename) ) self.insertSource(filename) else: if self.codelinePattern.match(text): self.sourcelines.append((text, line+1, pp_file)) labelPattern = re.compile('(\w+):\s+([^#\n\r]*)') opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE) varPattern = re.compile('var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}') # def parse(self): """ parse the code """ logger = logging.getLogger(__name__) self.code = [] self.variabledict = collections.OrderedDict() self.defines = dict() addr_offset = 0 for text, lineno, sourcename in self.sourcelines: m = self.varPattern.match(text) if m: self.addVariable(m, lineno, sourcename) else: m = self.definePattern.match(text) if m: self.addDefine(m, lineno, sourcename) else: # extract any JMP label, if present m = self.labelPattern.match(text) if m: label, text = m.groups() #so the operation after ":" still gets parsed CWC 08162012 else: label = None #The label for non-jump label line is NONE CWC 08172012 # search OPS list for a match to the current line m = self.opPattern.match(text) if m: op, args = m.groups() op = op.upper() # split and remove whitespace arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',')] #substitute the defined variable directly with the corresponding value CWC 08172012 arglist = [ self.defines[x] if x in self.defines else x for x in arglist ] #check for dds commands so CHAN commands can be inserted if (op[:3] == 'DDS'): try: board = self.adIndexList[int(arglist[0])][0] except ValueError: raise ppexception("DDS argument does not resolve to integer", sourcename, lineno, arglist[0]) chan = self.adIndexList[int(arglist[0])][1] if (self.adBoards[board].channelLimit != 1): #boards with more than one channel require an extra channel selection command chanData = self.adBoards[board].addCMD(chan) chanData += (int(board) << 16) self.code.append((len(self.code)+addr_offset, 'DDSCHN', chanData, label, sourcename, lineno)) data = arglist if len(arglist)>1 else arglist[0] self.addLabel( label, len(self.code), sourcename, lineno) self.code.append((len(self.code)+addr_offset, op, data, label, sourcename, lineno)) else: logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno) ) raise ppexception("Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno), sourcename, lineno, text) self.appendVariableCode() return self.code def appendVariableCode(self): """ append all variables to the instruction part of the code """ for var in list(self.variabledict.values()): address = len(self.code) self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0 )) var.address = address def addVariable(self, m, lineno, sourcename): """ add a variable to the self.variablesdict """ logger = logging.getLogger(__name__) logger.debug( "Variable {0} {1} {2}".format( m.groups(), lineno, sourcename ) ) var = Variable() label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups()] var.name = label var.origin = sourcename var.enabled = True if var.encoding not in encodings: raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, sourcename, lineno), sourcename, lineno, var.encoding) try: data = str(eval(data, globals(), self.defines)) except Exception: logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(sourcename, data) ) if unit is not None: var.value = Q(float(data), unit) data = self.convertParameter( var.value, var.encoding ) else: var.value = Q(float(data)) # var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :) data = int(round(float(data))) if label in self.defines: logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(sourcename, label, data, self.defines[label]) ) raise ppexception("variable redifinition", sourcename, lineno, label) else: self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name # however, we do not want it replaced with a number but keep the name for the last stage of compilation pass var.data = data self.variabledict.update({ label: var}) if var.type == "exitcode": self._exitcodes[data & 0x0000ffff] = var # code is (address, operation, data, label or variablename, currentfile) def toBytecode(self): """ generate bytecode from code """ logger = logging.getLogger(__name__) logger.debug( "\nCode ---> ByteCode:" ) self.bytecode = [] for line in self.code: logger.debug( "{0}: {1}".format(hex(line[0]), line[1:] )) bytedata = 0 if line[1] not in OPS: raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1]) byteop = OPS[line[1]] try: data = line[2] #attempt to locate commands with constant data if (data == ''): #found empty data bytedata = 0 elif isinstance(data, (int, int)): bytedata = data elif isinstance(data, float): bytedata = int(data) elif isinstance(data, str): # now we are dealing with a variable and need its address bytedata = self.variabledict[line[2]].address if line[2] in self.variabledict else self.labeldict[line[2]] elif isinstance(data, list): # list is what we have for DDS, will have 8bit channel and 16bit address channel, data = line[2] if isinstance(data, str): data = self.variabledict[data].address bytedata = ((int(channel) & 0xf) << 16) | (int(data) & 0x0fff) except KeyError: logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n".format(line[4], data) ) raise ppexception("{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data) self.bytecode.append((byteop, bytedata)) logger.debug( "---> {0} {1}".format(hex(byteop), hex(bytedata)) ) return self.bytecode def convertParameter(self, mag, encoding=None ): """ convert a dimensioned parameter to the binary value expected by the hardware. The conversion is determined by the variable encoding """ if is_Q(mag): if mag.dimensionality == Dimensions.time: result = int(round(mag / self.timestep)) else: step, unit, _, mask = encodings[encoding] result = int(round(mag.m_as(unit) / step)) & mask else: if encoding: step, unit, _, mask = encodings[encoding] result = int(round(mag/step)) & mask else: result = mag return result def compileCode(self): self.parse() self.toBytecode() def exitcode(self, code): if code in self._exitcodes: var = self._exitcodes[code] if var.comment: return var.comment else: return var.name else: return "Exitcode {0} Not found".format(code)