def read(self, file_or_filename):
""" Parses a PSAT data file and returns a case object
file_or_filename: File object or path to PSAT data file
return: Case object
"""
self.file_or_filename = file_or_filename
logger.info("Parsing PSAT case file [%s]." % file_or_filename)
t0 = time.time()
self.case = Case()
# Name the case
if isinstance(file_or_filename, basestring):
name, _ = splitext(basename(file_or_filename))
else:
name, _ = splitext(file_or_filename.name)
self.case.name = name
bus_array = self._get_bus_array_construct()
line_array = self._get_line_array_construct()
# TODO: Lines.con - Alternative line data format
slack_array = self._get_slack_array_construct()
pv_array = self._get_pv_array_construct()
pq_array = self._get_pq_array_construct()
demand_array = self._get_demand_array_construct()
supply_array = self._get_supply_array_construct()
# TODO: Varname.bus (Bus names)
# Pyparsing case:
case = \
ZeroOrMore(matlab_comment) + bus_array + \
ZeroOrMore(matlab_comment) + line_array + \
ZeroOrMore(matlab_comment) + slack_array + \
ZeroOrMore(matlab_comment) + pv_array + \
ZeroOrMore(matlab_comment) + pq_array + \
ZeroOrMore(matlab_comment) + demand_array + \
ZeroOrMore(matlab_comment) + supply_array
case.parseFile(file_or_filename)
elapsed = time.time() - t0
logger.info("PSAT case file parsed in %.3fs." % elapsed)
return self.case
def parse_cp2k_warnings(file_name, package_warnings):
"""
Parse All the warnings found in an output file
"""
p = ZeroOrMore(Suppress(SkipTo("*** WARNING")) + SkipTo('\n\n'))
# Return dict of Warnings
messages = p.parseFile(file_name).asList()
# Search for warnings that match the ones provided by the user
warnings = {m: assign_warning(package_warnings, m) for m in messages}
if not warnings:
return None
else:
return warnings
def parse_cp2k_warnings(file_name, package_warnings):
"""
Parse All the warnings found in an output file
"""
p = ZeroOrMore(Suppress(SkipTo("*** WARNING")) + SkipTo('\n\n'))
# Return dict of Warnings
messages = p.parseFile(file_name).asList()
# Search for warnings that match the ones provided by the user
warnings = {m: assign_warning(package_warnings, m) for m in messages}
if not warnings:
return None
else:
return warnings
def get_cp2k_thermo(file_name: PathLike, quantity: Quantity = 'G',
unit: str = 'kcal/mol') -> float:
"""Return thermochemical properties as extracted from a CP2K .out file.
Note
----
Note that CP2K can under certain circumstances report entropies and, by extension,
Gibbs free energies as :data:`~math.nan`.
Parameters
----------
file_name : :class:`str`, :class:`bytes` or :class:`os.PathLike`
A path-like object pointing to the CP2K .out file.
quantity : :class:`str`
The to-be returned quantity.
Accepted values are ``"E"``, ``"ZPE"``, ``"H"``, ``"S"`` and ``"G"``.
unit : :class:`str`
The unit of the to-be returned *quantity*.
See :class:`plams.Units<scm.plams.tools.units.Units>` for more details.
Returns
-------
:class:`float`
A user-specified *quantity* expressed in *unit* as extracted from *file_name*.
"""
quantity = quantity.upper()
if quantity not in QUANTITY_SET:
raise ValueError(f"'quantity' has an invalid value ({quantity!r}); "
f"expected values: {tuple(QUANTITY_SET)!r}")
parser = ZeroOrMore(Suppress(SkipTo(" VIB| Temperature ")) + SkipTo('\n\n\n\n'))
energy = next(iter(parser.parseFile(file_name)))
energy_iter = (i.rsplit(maxsplit=1) for i in energy.splitlines() if i)
energy_dict = {QUANTITY_MAPPING.get(k): float(v) for k, v in energy_iter}
energy_dict['H'] += energy_dict['E']
energy_dict['ZPE'] += energy_dict['E']
energy_dict['G'] = energy_dict['H'] - energy_dict['T'] * energy_dict['S']
return energy_dict[quantity] * Units.conversion_ratio('kj/mol', unit)
def parse(path):
# Comments
cmt = Suppress(Optional("," + CharsNotIn(",") + ","))
# Graph characters and construction
graphchars = "abcdefghijklmnopqrstuvwxyz"
graph = Word(graphchars)
# Input and output text
text = QuotedString("(", endQuoteChar=")", multiline=True)
inout = Optional(text, None)
# Command
cmd = cmt + inout + cmt + graph + cmt + inout + cmt + graph + cmt
cmd.setParseAction(lambda x, y, z: Command(*z))
# Program
prog = ZeroOrMore(cmd) + stringEnd
prog.setParseAction(lambda x, y, z: Program(z))
# Run parser
return prog.parseFile(path)[0]
def guess_language(string=None, filename=None):
"""
Attempt to guess the language
Do this by parsing the comments at the top of the file for the
# language: fr
phrase.
"""
LANG_PARSER = ZeroOrMore(
Suppress('#') + (
((Suppress(Keyword('language')) + Suppress(':') +
Word(unicodePrintables)('language')) |
Suppress(restOfLine))
)
)
try:
if string:
tokens = LANG_PARSER.parseString(string)
elif filename:
with open(filename, 'r', 'utf-8') as fp:
tokens = LANG_PARSER.parseFile(fp)
else:
raise RuntimeError("Must pass string or filename")
code = tokens.language
if code != '':
return languages.Language(code=code)
except ParseException as e:
# try English
pass
return languages.English()
'datetime': 'time_point',
'timestamp': 'time_point',
'enum': 'text', # MYSQL
'set': 'text', # MYSQL,
'tinyint unsigned': 'tinyint_unsigned', #MYSQL
'smallint unsigned': 'smallint_unsigned', #MYSQL
'integer unsigned': 'integer_unsigned', #MYSQL
'int unsigned': 'integer_unsigned', #MYSQL
'bigint unsigned': 'bigint_unsigned', #MYSQL
}
if failOnParse:
ddl = OneOrMore(Suppress(SkipTo(createTable, False)) + createTable)
ddl.ignore(ddlComment)
try:
tableCreations = ddl.parseFile(pathToDdl)
except ParseException as e:
print(parseError + '. Exiting [-no-fail-on-parse]')
sys.exit(ERROR_STRANGE_PARSING)
else:
ddl = ZeroOrMore(Suppress(SkipTo(createTable, False)) + createTable)
ddl.ignore(ddlComment)
tableCreations = ddl.parseFile(pathToDdl)
if warnOnParse:
print(parseError + '. Continuing [-no-warn-on-parse]')
nsList = namespace.split('::')
def escape_if_reserved(name):
def __init__(self, fragment_file, sdkconfig):
try:
fragment_file = open(fragment_file, "r")
except TypeError:
pass
path = os.path.realpath(fragment_file.name)
indent_stack = [1]
class parse_ctx:
fragment = None # current fragment
key = "" # current key
keys = list() # list of keys parsed
key_grammar = None # current key grammar
@staticmethod
def reset():
parse_ctx.fragment_instance = None
parse_ctx.key = ""
parse_ctx.keys = list()
parse_ctx.key_grammar = None
def fragment_type_parse_action(toks):
parse_ctx.reset()
parse_ctx.fragment = FRAGMENT_TYPES[
toks[0]]() # create instance of the fragment
return None
def expand_conditionals(toks, stmts):
try:
stmt = toks["value"]
stmts.append(stmt)
except KeyError:
try:
conditions = toks["conditional"]
for condition in conditions:
try:
_toks = condition[1]
_cond = condition[0]
if sdkconfig.evaluate_expression(_cond):
expand_conditionals(_toks, stmts)
break
except IndexError:
expand_conditionals(condition[0], stmts)
except KeyError:
for tok in toks:
expand_conditionals(tok, stmts)
def key_body_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
if parse_ctx.key_grammar.min and len(
stmts) < parse_ctx.key_grammar.min:
raise ParseFatalException(
pstr, loc,
"fragment requires at least %d values for key '%s'" %
(parse_ctx.key_grammar.min, parse_ctx.key))
if parse_ctx.key_grammar.max and len(
stmts) > parse_ctx.key_grammar.max:
raise ParseFatalException(
pstr, loc,
"fragment requires at most %d values for key '%s'" %
(parse_ctx.key_grammar.max, parse_ctx.key))
try:
parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to add key '%s'; %s" % (parse_ctx.key, e.message))
return None
key = Word(alphanums + "_") + Suppress(":")
key_stmt = Forward()
condition_block = indentedBlock(key_stmt, indent_stack)
key_stmts = OneOrMore(condition_block)
key_body = Suppress(key) + key_stmts
key_body.setParseAction(key_body_parsed)
condition = originalTextFor(
SDKConfig.get_expression_grammar()).setResultsName("condition")
if_condition = Group(
Suppress("if") + condition + Suppress(":") + condition_block)
elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + condition_block)
else_condition = Group(
Suppress("else") + Suppress(":") + condition_block)
conditional = (if_condition + Optional(OneOrMore(elif_condition)) +
Optional(else_condition)).setResultsName("conditional")
def key_parse_action(pstr, loc, toks):
key = toks[0]
if key in parse_ctx.keys:
raise ParseFatalException(
pstr, loc,
"duplicate key '%s' value definition" % parse_ctx.key)
parse_ctx.key = key
parse_ctx.keys.append(key)
try:
parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars(
)[key]
key_grammar = parse_ctx.key_grammar.grammar
except KeyError:
raise ParseFatalException(
pstr, loc, "key '%s' is not supported by fragment" % key)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to parse key '%s'; %s" % (key, e.message))
key_stmt << (conditional
| Group(key_grammar).setResultsName("value"))
return None
def name_parse_action(pstr, loc, toks):
parse_ctx.fragment.name = toks[0]
key.setParseAction(key_parse_action)
ftype = Word(alphas).setParseAction(fragment_type_parse_action)
fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
fid.setParseAction(name_parse_action)
header = Suppress("[") + ftype + fid + Suppress("]")
def fragment_parse_action(pstr, loc, toks):
key_grammars = parse_ctx.fragment.get_key_grammars()
required_keys = set(
[k for (k, v) in key_grammars.items() if v.required])
present_keys = required_keys.intersection(set(parse_ctx.keys))
if present_keys != required_keys:
raise ParseFatalException(
pstr, loc, "required keys %s for fragment not found" %
list(required_keys - present_keys))
return parse_ctx.fragment
fragment_stmt = Forward()
fragment_block = indentedBlock(fragment_stmt, indent_stack)
fragment_if_condition = Group(
Suppress("if") + condition + Suppress(":") + fragment_block)
fragment_elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + fragment_block)
fragment_else_condition = Group(
Suppress("else") + Suppress(":") + fragment_block)
fragment_conditional = (
fragment_if_condition +
Optional(OneOrMore(fragment_elif_condition)) +
Optional(fragment_else_condition)).setResultsName("conditional")
fragment = (header +
OneOrMore(indentedBlock(key_body, indent_stack,
False))).setResultsName("value")
fragment.setParseAction(fragment_parse_action)
fragment.ignore("#" + restOfLine)
deprecated_mapping = DeprecatedMapping.get_fragment_grammar(
sdkconfig, fragment_file.name).setResultsName("value")
fragment_stmt << (Group(deprecated_mapping) | Group(fragment)
| Group(fragment_conditional))
def fragment_stmt_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
return stmts
parser = ZeroOrMore(fragment_stmt)
parser.setParseAction(fragment_stmt_parsed)
self.fragments = parser.parseFile(fragment_file, parseAll=True)
for fragment in self.fragments:
fragment.path = path
class pppCompiler:
def __init__(self):
self.initBNF()
self.symbols = SymbolTable()
def initBNF(self):
indentStack = [1]
encoding = Literal("<").suppress() + identifier("encoding") + Literal(
">").suppress()
constdecl = Group((const + identifier + assign + value).setParseAction(
self.const_action))
vardecl = Group(
(type_("type_") + Optional(encoding) + identifier("name") +
Optional(assign + value("value") + Optional(identifier)("unit"))
).setParseAction(self.var_action))
insertdecl = Group(
(insert + dblQuotedString + LineEnd().suppress()).setParseAction(
self.insert_action))
procedurecall = Group((identifier + Literal("(").suppress() + Optional(
delimitedList(
(identifier + Optional(assign + identifier)).setParseAction(
self.named_param_action))) +
Literal(")").suppress()).setParseAction(
self.procedurecall_action))
condition = Group(
(identifier("leftidentifier") + comparison("comparison") +
(identifier("identifier")
| value.setParseAction(self.value_action))).setParseAction(
self.condition_action))("condition")
pointer = Literal("*") + identifier
rExp = Forward()
#numexpression = Forward()
opexpression = (identifier("operand") +
(Literal(">>") | Literal("<<") | Literal("+")
| Literal("*") | Literal("/") | Literal("-"))("op") +
Group(rExp)("argument")).setParseAction(
self.opexpression_action)
rExp << (
procedurecall | opexpression | identifier("identifier")
| value.setParseAction(self.value_action) |
#Group( Suppress("(") + rExp + Suppress(")") ) |
#Group( "+" + rExp) |
#Group( "-" + rExp) |
Group(Literal("not") + rExp))
rExpCondition = Group(
(Optional(not_)("not_") + rExp("rExp"))).setParseAction(
self.rExp_condition_action)("condition")
rExp.setParseAction(self.rExp_action)
assignment = ((identifier | pointer)("lval") + assign +
rExp("rval")).setParseAction(self.assignment_action)
addassignment = ((identifier | pointer)("lval") +
(Literal("+=") | Literal("-=") | Literal("*=")
| Literal("&=") | Literal("|=") | Literal(">>=")
| Literal("/=") | Literal("<<="))("op") +
Group(rExp)("rval")).setParseAction(
self.addassignement_action)
statement = Forward()
statementBlock = indentedBlock(statement, indentStack).setParseAction(
self.statementBlock_action)
procedure_statement = Group(
(Keyword("def").suppress() + identifier("funcname") +
Literal("(").suppress() + Literal(")").suppress() +
colon.suppress() + statementBlock).setParseAction(
self.def_action))
while_statement = Group(
(Keyword("while").suppress() +
(condition | rExpCondition)("condition") + colon.suppress() +
statementBlock("statementBlock")).setParseAction(
self.while_action))
if_statement = (Keyword("if") + condition + colon +
statementBlock("ifblock") + Optional(
Keyword("else").suppress() + colon +
statementBlock("elseblock"))).setParseAction(
self.if_action)
statement << (procedure_statement | while_statement | if_statement
| procedurecall | assignment | addassignment)
decl = constdecl | vardecl | insertdecl | Group(statement)
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def assignment_action(self, text, loc, arg):
logging.getLogger(__name__).debug("assignment_action {0} {1}".format(
lineno(loc, text), arg))
try:
code = [
"# line {0} assignment {1}".format(lineno(loc, text),
line(loc, text))
]
rval_code = find_and_get(arg.rval, 'code')
if rval_code is not None:
code += arg.rval.code
elif arg.rval == "*P":
code.append(" LDWI")
elif 'identifier' in arg:
self.symbols.getVar(arg.identifier)
code.append(" LDWR {0}".format(arg.identifier))
if arg.lval == "*P":
code.append(" STWI")
elif arg.lval != "W":
symbol = self.symbols.getVar(arg.lval)
code.append(" STWR {0}".format(symbol.name))
if 'code' in arg:
arg['code'].extend(code)
else:
arg['code'] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def addassignement_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"addassignement_action {0} {1}".format(lineno(loc, text), arg))
try:
code = [
"# line {0}: add_assignment: {1}".format(
lineno(loc, text), line(loc, text))
]
if arg.rval[0] == '1' and arg.op in ['+=', '-=']:
self.symbols.getVar(arg.lval)
if arg.op == "+=":
code.append(" INC {0}".format(arg.lval))
else:
code.append(" DEC {0}".format(arg.lval))
else:
if 'code' in arg.rval:
code += arg.rval.code
self.symbols.getVar(arg.lval)
if arg.op == "-=":
raise CompileException(
"-= with expression needs to be fixed in the compiler"
)
code.append(" {0} {1}".format(opassignmentLookup[arg.op],
arg.lval))
elif 'identifier' in arg.rval:
self.symbols.getVar(arg.rval.identifier)
code.append(" LDWR {0}".format(arg.lval))
self.symbols.getVar(arg.lval)
code.append(" {0} {1}".format(opassignmentLookup[arg.op],
arg.rval.identifier))
code.append(" STWR {0}".format(arg.lval))
arg['code'] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def condition_action(self, text, loc, arg):
logging.getLogger(__name__).debug("condition_action {0} {1}".format(
lineno(loc, text), arg))
try:
code = [
"# line {0} condition {1}".format(lineno(loc, text),
line(loc, text))
]
if arg.leftidentifier != "W":
self.symbols.getVar(arg.leftidentifier)
code.append(' LDWR {0}'.format(arg.leftidentifier))
if arg.identifier == 'NULL' and arg.comparison in jmpNullCommands:
arg['jmpcmd'] = jmpNullCommands[arg.comparison]
else:
code.append(' {0} {1}'.format(
comparisonCommands[arg.comparison], arg.identifier))
arg["code"] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def rExp_condition_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"rExp_condition_action {0} {1}".format(lineno(loc, text), arg))
try:
code = [
"# line {0} rExp_condition {1}".format(lineno(loc, text),
line(loc, text))
]
condition_code = arg.condition.rExp['code']
if isinstance(condition_code, str):
if 'not_' in arg['condition']:
code += [" CMPEQUAL NULL"]
else:
code += [" CMPNOTEQUAL NULL"]
arg['code'] = code
else:
if 'not_' in arg['condition']:
arg['code'] = {
False: condition_code[True],
True: condition_code[False]
}
else:
arg['code'] = condition_code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def named_param_action(self, text, loc, arg):
if len(arg) == 2:
arg[arg[0]] = arg[1]
return arg
def value_action(self, text, loc, arg):
if arg[0][0:2] == '0x':
value = int(arg[0], 16)
else:
value = int(arg[0])
arg["identifier"] = self.symbols.getInlineParameter("inlinevar", value)
return arg
def opexpression_action(self, text, loc, arg):
try:
logging.getLogger(__name__).debug(
"opexpression_action {0} {1}".format(lineno(loc, text), arg))
code = [
"# line {0}: shiftexpression {1}".format(
lineno(loc, text),
line(loc, text)), " LDWR {0}".format(arg.operand),
" {0} {1}".format(shiftLookup[arg.op],
arg.argument.identifier)
]
arg['code'] = code
logging.getLogger(__name__).debug(
"shiftexpression generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def procedurecall_action(self, text, loc, arg):
try:
logging.getLogger(__name__).debug(
"procedurecall_action {0} {1}".format(lineno(loc, text), arg))
procedure = self.symbols.getProcedure(arg[0])
code = [
"# line {0}: procedurecall {1}".format(lineno(loc, text),
line(loc, text))
]
opcode = procedure.codegen(self.symbols,
arg=arg.asList(),
kwarg=arg.asDict())
if isinstance(opcode, list):
code += opcode
else:
code = opcode
arg['code'] = code
logging.getLogger(__name__).debug(
"procedurecall generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def rExp_action(self, text, loc, arg):
logging.getLogger(__name__).debug("rExp_action {0} {1}".format(
lineno(loc, text), arg))
pass
def if_action(self, text, loc, arg):
logging.getLogger(__name__).debug("if_action {0} {1}".format(
lineno(loc, text), arg))
try:
block0 = [
"# line {0} if statement {1}".format(lineno(loc, text),
line(loc, text))
]
if isinstance(arg.condition.code, list):
block0 += arg.condition.code
JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False]
else:
JMPCMD = arg.condition.code[True]
if 'elseblock' in arg:
block1 = arg.ifblock.ifblock.code
block2 = arg.elseblock.elseblock[
'code'] if 'elseblock' in arg.elseblock else arg.elseblock[
'code']
else:
block1 = arg.ifblock.ifblock['code']
block2 = None
arg['code'] = [
IfGenerator(self.symbols, JMPCMD, block0, block1, block2)
]
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def while_action(self, text, loc, arg):
logging.getLogger(__name__).debug("while_action {0} {1}".format(
lineno(loc, text), arg))
try:
block0 = [
"# line {0} while_statement {1}".format(
lineno(loc, text), line(loc, text))
]
if 'code' in arg.condition:
if isinstance(arg.condition.code, list):
block1 = arg.condition.code
JMPCMD = arg.condition.get('jmpcmd',
{False: "JMPNCMP"})[False]
else:
JMPCMD = arg.condition.code[True]
block1 = []
elif 'rExp' in arg.condition and 'code' in arg.condition.rExp:
if isinstance(arg.condition.rExp.code, list):
block1 = arg.condition.rExp.code
JMPCMD = arg.condition.rExp.get('jmpcmd', "JMPNCMP")
else:
JMPCMD = arg.condition.rExp.code[True]
block1 = []
block2 = arg.statementBlock.statementBlock['code']
arg['code'] = [
WhileGenerator(self.symbols, JMPCMD, block0, block1, block2)
]
logging.getLogger(__name__).debug("while_action generated code ")
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def statementBlock_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"statementBlock_action {0} {1} {2}".format(lineno(loc, text),
arg.funcname, arg))
try:
code = list()
for command in arg[0]:
if 'code' in command:
code += command['code']
elif 'code' in command[0]:
code += command[0]['code']
arg[0]['code'] = code
logging.getLogger(__name__).debug(
"statementBlock generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def def_action(self, text, loc, arg):
logging.getLogger(__name__).debug("def_action {0} {1} {2}".format(
lineno(loc, text), arg.funcname, arg))
try:
name = arg[0]
self.symbols.checkAvailable(name)
self.symbols[name] = FunctionSymbol(name, arg[1]['code'])
except Exception as e:
raise CompileException(text, loc, str(e), self)
def const_action(self, text, loc, arg):
try:
name, value = arg
logging.getLogger(__name__).debug(
"const_action {0} {1} {2} {3}".format(self.currentFile,
lineno(loc, text), name,
value))
self.symbols[name] = ConstSymbol(name, value)
except Exception as e:
raise CompileException(text, loc, str(e), self)
def var_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"var_action {0} {1} {2} {3} {4} {5} {6}".format(
self.currentFile, lineno(loc, text), arg["type_"],
arg.get("encoding"), arg["name"], arg.get("value"),
arg.get("unit")))
try:
type_ = arg["type_"] if arg["type_"] != "var" else None
self.symbols[arg["name"]] = VarSymbol(type_=type_,
name=arg["name"],
value=arg.get("value"),
encoding=arg.get("encoding"),
unit=arg.get("unit"))
except Exception as e:
raise CompileException(text, loc, str(e), self)
def insert_action(self, text, loc, arg):
try:
oldfile = self.currentFile
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile(self.currentFile)
self.currentFile = oldfile
except Exception as e:
raise CompileException(text, loc, str(e), self)
return result
def compileFile(self, filename):
self.currentFile = filename
result = self.program.parseFile(self.currentFile, parseAll=True)
allcode = list()
for element in result:
if not isinstance(element, str) and 'code' in element:
allcode += element['code']
elif not isinstance(element[0], str) and 'code' in element[0]:
allcode += element[0]['code']
header = self.createHeader()
codetext = "\n".join(header + allcode)
return codetext
def compileString(self, programText):
self.programText = programText
self.currentFile = "Memory"
result = self.program.parseString(self.programText, parseAll=True)
allcode = list()
for element in result:
if not isinstance(element, str) and 'code' in element:
allcode += element['code']
elif not isinstance(element[0], str) and 'code' in element[0]:
allcode += element[0]['code']
header = self.createHeader()
codetext = """# autogenerated
# DO NOT EDIT DIRECTLY
# The file will be overwritten by the compiler
#
"""
codetext += "\n".join(header + list(generate(allcode)))
self.reverseLineLookup = self.generateReverseLineLookup(codetext)
return codetext
def generateReverseLineLookup(self, codetext):
lookup = dict()
sourceline = None
for codeline, line in enumerate(codetext.splitlines()):
m = re.search('^\# line (\d+).*$', line)
if m:
sourceline = int(m.group(1))
else:
lookup[codeline + 1] = sourceline
return lookup
def createHeader(self):
header = ["# const values"]
for constval in self.symbols.getAllConst():
header.append("const {0} {1}".format(constval.name,
constval.value))
header.append("# variables ")
for var in self.symbols.getAllVar():
if var.type_ == "masked_shutter":
header.append("var {0} {1}, {2}".format(
var.name + "_mask",
var.value if var.value is not None else 0, "mask"))
header.append("var {0} {1}, {2}".format(
var.name, var.value if var.value is not None else 0,
"shutter {0}_mask".format(var.name)))
else:
optionals = [
s if s is not None else ""
for s in list_rtrim([var.type_, var.unit, var.encoding])
]
varline = "var {0} {1}".format(
var.name, var.value if var.value is not None else 0)
if len(optionals) > 0:
varline += ", " + ", ".join(optionals)
header.append(varline)
header.append("# inline variables")
# for value, name in self.symbols.inlineParameterValues.items():
# header.append("var {0} {1}".format(name, value))
header.append("# end header")
header.append("")
return header
pathToHeader = sys.argv[2] + '.h'
namespace = sys.argv[3]
ddlFile = open(pathToDdl, 'r')
header = open(pathToHeader, 'w')
print('#ifndef '+get_include_guard_name(namespace, pathToHeader), file=header)
print('#define '+get_include_guard_name(namespace, pathToHeader), file=header)
print('', file=header)
print('#include <' + INCLUDE + '/table.h>', file=header)
print('#include <' + INCLUDE + '/column_types.h>', file=header)
print('#include <' + INCLUDE + '/char_sequence.h>', file=header)
print('', file=header)
print('namespace ' + namespace, file=header)
print('{', file=header)
tableCreations = ddl.parseFile(pathToDdl)
for tableCreation in tableCreations:
sqlTableName = tableCreation.create.tableName
tableClass = toClassName(sqlTableName)
tableMember = toMemberName(sqlTableName)
tableNamespace = tableClass + '_'
tableTemplateParameters = tableClass
print(' namespace ' + tableNamespace, file=header)
print(' {', file=header)
for column in tableCreation.create.columns:
if column.isConstraint:
continue
sqlColumnName = column[0]
columnClass = toClassName(sqlColumnName)
tableTemplateParameters += ',\n ' + tableNamespace + '::' + columnClass
record = (recordA_ ^
recordSOA_ ^
recordNS_ ^
recordMX_ ^
recordCNAME_ ^
recordTXT_ ^
recordSRV_ ^
recordPTR_ ^
recordRP_ ^
recordLOC_ ^
recordNAPTR_)
lines = (Group(origin | ttl)('DIRECTIVE') |
comment |
record)
zoneParser = ZeroOrMore(lines)('zone') + stringEnd
if __name__ == '__main__':
if len(sys.argv) != 2:
print "Usage: {} <zone_file>".format(sys.argv[0])
sys.exit(-1)
zone_file = sys.argv[1]
try:
tokens = zoneParser.parseFile(zone_file)
print tokens.asXML()
except ParseException:
raise
class PulseProgram:
""" Encapsulates a PulseProgrammer Program
loadSource( filename ) loads the contents of the file
The code is compiled in the following steps
parse() generates self.code
toBytecode() generates self.bytecode
toBinary() generates self.binarycode
the procedure updateVariables( dictionary ) updates variable values in the bytecode
"""
def __init__(self):
self.variabledict = collections.OrderedDict(
) # keeps information on all variables to easily change them later
self.labeldict = dict() # keep information on all labels
self.source = collections.OrderedDict(
) # dictionary of source code files (stored as strings)
self.code = [] # this is a list of lines
self.bytecode = [] # list of op, argument tuples
self.binarycode = bytearray() # binarycode to be uploaded
self._exitcodes = dict(
) # generate a reverse dictionary of variables of type exitcode
self.constDict = dict()
class Board:
channelLimit = 1
halfClockLimit = 500000000
self.adIndexList = [(x, 0) for x in range(6)]
self.adBoards = [Board()] * 6
self.timestep = Q(20.0, 'ns')
self.initBNF()
def initBNF(self):
constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action)
vardecl = (VAR + NAME + VALUE +
Optional(COMMA + Regex("[^#\n]*"))).setParseAction(
self.var_action)
insertdecl = (INSERT + dblQuotedString +
LineEnd().suppress()).setParseAction(self.insert_action)
LABEL = IDENTIFIER + COLON
COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") +
Regex("[^#\n]*").setWhitespaceChars(" \t") +
LineEnd().suppress())
COMMAND = COMMANDEXP.setParseAction(self.command_action)
LABELEDCOMMAND = (LABEL + COMMANDEXP).setParseAction(
self.label_command_action)
decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def const_action(self, text, loc, arg):
""" add the const to the self.constDict dictionary
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} const {2}".format(self.currentFile,
lineno(loc, text), arg))
label, value = arg
if label in self.constDict:
logger.error(
"Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'"
.format(self.currentFile, label, value, self.constDict[label]))
raise ppexception("Redefining variable", self.currentFile, lineno,
label)
else:
self.constDict[label] = int(value)
def var_action(self, text, loc, arg):
print("var_action", self.currentFile, lineno(loc, text), arg[0:2],
arg[2].split(",") if len(arg) > 2 else "")
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} Variable {2}".format(self.currentFile,
lineno(loc, text), arg))
var = Variable()
label, data = arg[:2]
fields = arg[2].split(",") if len(arg) > 2 else [None] * 3
fields += [None] * (3 - len(fields))
var.type, unit, var.encoding = [
x if x is None or '' else x.strip() for x in fields
]
var.name = label
var.origin = self.currentFile
var.enabled = True
if var.encoding not in encodings:
raise ppexception(
"unknown encoding {0} in file '{1}':{2}".format(
var.encoding, self.currentFile, lineno(loc, text)),
self.currentFile, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception(
"Evaluation error in file '{0}' on line: '{1}'".format(
self.currentFile, data))
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter(var.value, var.encoding)
else:
var.value = Q(float(data))
data = int(round(float(data)))
if label in self.defines:
logger.error(
"Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement."
% (self.currentFile, label, data, self.defines[label]))
raise ppexception("variable redifinition", self.currentFile,
lineno, label)
else:
self.defines[
label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
def command_action(self, text, loc, arg):
print("command_action", self.currentFile, lineno(loc, text), arg[0:1],
arg[1].split(",") if len(arg) > 1 else "")
def label_command_action(self, text, loc, arg):
print("label_command_action", self.currentFile, lineno(loc, text),
arg[0:2], arg[2].split(",") if len(arg) > 2 else "")
def addLabel(self, label, address, sourcename, lineno):
if label is not None:
self.labeldict[label] = address
def insert_action(self, text, loc, arg):
oldfile = self.currentFile
print("insert_action", lineno(loc, text), arg)
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile(self.currentFile)
self.currentFile = oldfile
print(result)
return result
def assembleFile(self, filename):
self.currentFile = filename
result = self.program.parseFile(self.currentFile)
return result
def setHardware(self, adIndexList, adBoards, timestep):
self.adIndexList = adIndexList
self.adBoards = adBoards
self.timestep = timestep
assert self.timestep.has_dimension('s')
def saveSource(self):
for name, text in self.source.items():
with open(os.path.join(self.pp_dir, name), 'w') as f:
f.write(text)
def loadSource(self, pp_file):
""" Load the source pp_file
#include files are loaded recursively
all code lines are added to self.sourcelines
for each source file the contents are added to the dictionary self.source
"""
self.source.clear()
self.pp_dir, self.pp_filename = os.path.split(pp_file)
self.sourcelines = []
self.insertSource(self.pp_filename)
self.compileCode()
def updateVariables(self, variables):
""" update the variable values in the bytecode
"""
logger = logging.getLogger(__name__)
for name, value in variables.items():
if name in self.variabledict:
var = self.variabledict[name]
address = var.address
var.value = value
logger.debug(
"updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}"
.format(name, value, address, int(var.data)))
var.data = self.convertParameter(value, var.encoding)
self.bytecode[address] = (self.bytecode[address][0], var.data)
self.variabledict[name] = var
else:
logger.error(
"variable {0} not found in dictionary.".format(name))
return self.bytecode
def variables(self):
mydict = dict()
for name, var in self.variabledict.items():
mydict.update({name: var.value})
return mydict
def variable(self, variablename):
return self.variabledict.get(variablename).value
def variableUpdateCode(self, variablename, value):
"""returns the code to update the variable directly on the fpga
consists of variablelocation and variablevalue
"""
var = self.variabledict[variablename]
data = self.convertParameter(value, var.encoding)
return bytearray(struct.pack('II', (var.address, data)))
def flattenList(self, l):
return [item for sublist in l for item in sublist]
def variableScanCode(self, variablename, values):
var = self.variabledict[variablename]
# [item for sublist in l for item in sublist] idiom for flattening of list
return self.flattenList([(var.address,
self.convertParameter(x, var.encoding))
for x in values])
def loadFromMemory(self):
"""Similar to loadSource
only this routine loads from self.source
"""
self.sourcelines = []
self._exitcodes = dict()
self.insertSource(self.pp_filename)
self.compileCode()
def toBinary(self):
""" convert bytecode to binary
"""
logger = logging.getLogger(__name__)
self.binarycode = bytearray()
for wordno, (op, arg) in enumerate(self.bytecode):
logger.debug("{0} {1} {2} {3}".format(
hex(wordno), hex(int(op)), hex(int(arg)),
hex(int((int(op) << 24) + int(arg)))))
self.binarycode += struct.pack('I', int((op << 24) + arg))
return self.binarycode
def currentVariablesText(self):
lines = list()
for name, var in iter(sorted(self.variabledict.items())):
lines.append("{0} {1}".format(name, var.value))
return '\n'.join(lines)
# routines below here should not be needed by the user
insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE)
codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE)
def insertSource(self, pp_file):
""" read a source file pp_file
calls itself recursively to for #insert
adds the contents of this file to the dictionary self.source
"""
logger = logging.getLogger(__name__)
if pp_file not in self.source:
with open(os.path.join(self.pp_dir, pp_file)) as f:
self.source[pp_file] = ''.join(f.readlines())
sourcecode = self.source[pp_file]
for line, text in enumerate(sourcecode.splitlines()):
m = self.insertPattern.match(text)
if m:
filename = m.group(1)
logger.info("inserting code from {0}".format(filename))
self.insertSource(filename)
else:
if self.codelinePattern.match(text):
self.sourcelines.append((text, line + 1, pp_file))
labelPattern = re.compile('(\w+):\s+([^#\n\r]*)')
opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE)
varPattern = re.compile(
'var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}'
) #
def parse(self):
""" parse the code
"""
logger = logging.getLogger(__name__)
self.code = []
self.variabledict = collections.OrderedDict()
self.defines = dict()
addr_offset = 0
for text, lineno, sourcename in self.sourcelines:
m = self.varPattern.match(text)
if m:
self.addVariable(m, lineno, sourcename)
else:
m = self.definePattern.match(text)
if m:
self.addDefine(m, lineno, sourcename)
else:
# extract any JMP label, if present
m = self.labelPattern.match(text)
if m:
label, text = m.groups(
) #so the operation after ":" still gets parsed CWC 08162012
else:
label = None #The label for non-jump label line is NONE CWC 08172012
# search OPS list for a match to the current line
m = self.opPattern.match(text)
if m:
op, args = m.groups()
op = op.upper()
# split and remove whitespace
arglist = [0] if args is None else [
0 if x is None else x.strip()
for x in args.split(',')
]
#substitute the defined variable directly with the corresponding value CWC 08172012
arglist = [
self.defines[x] if x in self.defines else x
for x in arglist
]
#check for dds commands so CHAN commands can be inserted
if (op[:3] == 'DDS'):
try:
board = self.adIndexList[int(arglist[0])][0]
except ValueError:
raise ppexception(
"DDS argument does not resolve to integer",
sourcename, lineno, arglist[0])
chan = self.adIndexList[int(arglist[0])][1]
if (self.adBoards[board].channelLimit != 1):
#boards with more than one channel require an extra channel selection command
chanData = self.adBoards[board].addCMD(chan)
chanData += (int(board) << 16)
self.code.append(
(len(self.code) + addr_offset, 'DDSCHN',
chanData, label, sourcename, lineno))
data = arglist if len(arglist) > 1 else arglist[0]
self.addLabel(label, len(self.code), sourcename,
lineno)
self.code.append((len(self.code) + addr_offset, op,
data, label, sourcename, lineno))
else:
logger.error(
"Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)"
.format(text, sourcename, lineno))
raise ppexception(
"Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)"
.format(text, sourcename,
lineno), sourcename, lineno, text)
self.appendVariableCode()
return self.code
def appendVariableCode(self):
""" append all variables to the instruction part of the code
"""
for var in list(self.variabledict.values()):
address = len(self.code)
self.code.append((address, 'NOP', var.data if var.enabled else 0,
None, var.origin, 0))
var.address = address
def addVariable(self, m, lineno, sourcename):
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug("Variable {0} {1} {2}".format(m.groups(), lineno,
sourcename))
var = Variable()
label, data, var.type, unit, var.encoding, var.comment = [
x if x is None else x.strip() for x in m.groups()
]
var.name = label
var.origin = sourcename
var.enabled = True
if var.encoding not in encodings:
raise ppexception(
"unknown encoding {0} in file '{1}':{2}".format(
var.encoding, sourcename, lineno), sourcename, lineno,
var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception(
"Evaluation error in file '{0}' on line: '{1}'".format(
sourcename, data))
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter(var.value, var.encoding)
else:
var.value = Q(float(data))
# var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :)
data = int(round(float(data)))
if label in self.defines:
logger.error(
"Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement."
% (sourcename, label, data, self.defines[label]))
raise ppexception("variable redifinition", sourcename, lineno,
label)
else:
self.defines[
label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
# code is (address, operation, data, label or variablename, currentfile)
def toBytecode(self):
""" generate bytecode from code
"""
logger = logging.getLogger(__name__)
logger.debug("\nCode ---> ByteCode:")
self.bytecode = []
for line in self.code:
logger.debug("{0}: {1}".format(hex(line[0]), line[1:]))
bytedata = 0
if line[1] not in OPS:
raise ppexception("Unknown command {0}".format(line[1]),
line[4], line[5], line[1])
byteop = OPS[line[1]]
try:
data = line[2]
#attempt to locate commands with constant data
if (data == ''):
#found empty data
bytedata = 0
elif isinstance(data, (int, int)):
bytedata = data
elif isinstance(data, float):
bytedata = int(data)
elif isinstance(
data, str
): # now we are dealing with a variable and need its address
bytedata = self.variabledict[line[2]].address if line[
2] in self.variabledict else self.labeldict[line[2]]
elif isinstance(
data, list
): # list is what we have for DDS, will have 8bit channel and 16bit address
channel, data = line[2]
if isinstance(data, str):
data = self.variabledict[data].address
bytedata = (
(int(channel) & 0xf) << 16) | (int(data) & 0x0fff)
except KeyError:
logger.error(
"Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n"
.format(line[4], data))
raise ppexception(
"{0}: Unknown variable {1}".format(line[4], data), line[4],
line[5], data)
self.bytecode.append((byteop, bytedata))
logger.debug("---> {0} {1}".format(hex(byteop), hex(bytedata)))
return self.bytecode
def convertParameter(self, mag, encoding=None):
""" convert a dimensioned parameter to the binary value
expected by the hardware. The conversion is determined by the variable encoding
"""
if is_Q(mag):
if mag.dimensionality == Dimensions.time:
result = int(round(mag / self.timestep))
else:
step, unit, _, mask = encodings[encoding]
result = int(round(mag.m_as(unit) / step)) & mask
else:
if encoding:
step, unit, _, mask = encodings[encoding]
result = int(round(mag / step)) & mask
else:
result = mag
return result
def compileCode(self):
self.parse()
self.toBytecode()
def exitcode(self, code):
if code in self._exitcodes:
var = self._exitcodes[code]
if var.comment:
return var.comment
else:
return var.name
else:
return "Exitcode {0} Not found".format(code)
class PulseProgram:
""" Encapsulates a PulseProgrammer Program
loadSource( filename ) loads the contents of the file
The code is compiled in the following steps
parse() generates self.code
toBytecode() generates self.bytecode
toBinary() generates self.binarycode
the procedure updateVariables( dictionary ) updates variable values in the bytecode
"""
def __init__(self):
self.variabledict = collections.OrderedDict() # keeps information on all variables to easily change them later
self.labeldict = dict() # keep information on all labels
self.source = collections.OrderedDict() # dictionary of source code files (stored as strings)
self.code = [] # this is a list of lines
self.bytecode = [] # list of op, argument tuples
self.binarycode = bytearray() # binarycode to be uploaded
self._exitcodes = dict() # generate a reverse dictionary of variables of type exitcode
self.constDict = dict()
class Board:
channelLimit = 1
halfClockLimit = 500000000
self.adIndexList = [(x, 0) for x in range(6) ]
self.adBoards = [ Board() ]*6
self.timestep = Q(20.0, 'ns')
self.initBNF()
def initBNF(self):
constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action)
vardecl = (VAR + NAME + VALUE + Optional( COMMA + Regex("[^#\n]*")) ).setParseAction(self.var_action)
insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action)
LABEL = IDENTIFIER + COLON
COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress() )
COMMAND = COMMANDEXP.setParseAction(self.command_action)
LABELEDCOMMAND = (LABEL + COMMANDEXP ).setParseAction(self.label_command_action)
decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def const_action( self, text, loc, arg ):
""" add the const to the self.constDict dictionary
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg))
label, value = arg
if label in self.constDict:
logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'".format(self.currentFile, label, value, self.constDict[label]) )
raise ppexception("Redefining variable", self.currentFile, lineno, label)
else:
self.constDict[label] = int(value)
def var_action( self, text, loc, arg):
print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "")
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug( "{0}:{1} Variable {2}".format( self.currentFile, lineno(loc, text), arg ) )
var = Variable()
label, data = arg[:2]
fields = arg[2].split(",") if len(arg)>2 else [None]*3
fields += [None]*(3-len(fields))
var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ]
var.name = label
var.origin = self.currentFile
var.enabled = True
if var.encoding not in encodings:
raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(self.currentFile, data) )
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter( var.value, var.encoding )
else:
var.value = Q(float(data))
data = int(round(float(data)))
if label in self.defines:
logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(self.currentFile, label, data, self.defines[label]) )
raise ppexception("variable redifinition", self.currentFile, lineno, label)
else:
self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({ label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
def command_action( self, text, loc, arg):
print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg)>1 else "")
def label_command_action( self, text, loc, arg):
print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "")
def addLabel(self, label, address, sourcename, lineno):
if label is not None:
self.labeldict[label] = address
def insert_action( self, text, loc, arg ):
oldfile = self.currentFile
print("insert_action", lineno(loc, text), arg)
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile( self.currentFile )
self.currentFile = oldfile
print(result)
return result
def assembleFile(self, filename):
self.currentFile = filename
result = self.program.parseFile( self.currentFile )
return result
def setHardware(self, adIndexList, adBoards, timestep ):
self.adIndexList = adIndexList
self.adBoards = adBoards
self.timestep = timestep
assert self.timestep.has_dimension('s')
def saveSource(self):
for name, text in self.source.items():
with open(os.path.join(self.pp_dir, name), 'w') as f:
f.write(text)
def loadSource(self, pp_file):
""" Load the source pp_file
#include files are loaded recursively
all code lines are added to self.sourcelines
for each source file the contents are added to the dictionary self.source
"""
self.source.clear()
self.pp_dir, self.pp_filename = os.path.split(pp_file)
self.sourcelines = []
self.insertSource(self.pp_filename)
self.compileCode()
def updateVariables(self, variables ):
""" update the variable values in the bytecode
"""
logger = logging.getLogger(__name__)
for name, value in variables.items():
if name in self.variabledict:
var = self.variabledict[name]
address = var.address
var.value = value
logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}".format(name, value, address, int(var.data)) )
var.data = self.convertParameter(value, var.encoding )
self.bytecode[address] = (self.bytecode[address][0], var.data )
self.variabledict[name] = var
else:
logger.error( "variable {0} not found in dictionary.".format(name) )
return self.bytecode
def variables(self):
mydict = dict()
for name, var in self.variabledict.items():
mydict.update( {name: var.value })
return mydict
def variable(self, variablename ):
return self.variabledict.get(variablename).value
def variableUpdateCode(self, variablename, value ):
"""returns the code to update the variable directly on the fpga
consists of variablelocation and variablevalue
"""
var = self.variabledict[variablename]
data = self.convertParameter(value, var.encoding )
return bytearray( struct.pack('II', (var.address, data)))
def flattenList(self, l):
return [item for sublist in l for item in sublist]
def variableScanCode(self, variablename, values):
var = self.variabledict[variablename]
# [item for sublist in l for item in sublist] idiom for flattening of list
return self.flattenList( [ (var.address, self.convertParameter(x, var.encoding)) for x in values ] )
def loadFromMemory(self):
"""Similar to loadSource
only this routine loads from self.source
"""
self.sourcelines = []
self._exitcodes = dict()
self.insertSource(self.pp_filename)
self.compileCode()
def toBinary(self):
""" convert bytecode to binary
"""
logger = logging.getLogger(__name__)
self.binarycode = bytearray()
for wordno, (op, arg) in enumerate(self.bytecode):
logger.debug( "{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op)<<24) + int(arg))) ) )
self.binarycode += struct.pack('I', int((op<<24) + arg))
return self.binarycode
def currentVariablesText(self):
lines = list()
for name, var in iter(sorted(self.variabledict.items())):
lines.append("{0} {1}".format(name, var.value))
return '\n'.join(lines)
# routines below here should not be needed by the user
insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE)
codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE)
def insertSource(self, pp_file):
""" read a source file pp_file
calls itself recursively to for #insert
adds the contents of this file to the dictionary self.source
"""
logger = logging.getLogger(__name__)
if pp_file not in self.source:
with open(os.path.join(self.pp_dir, pp_file)) as f:
self.source[pp_file] = ''.join(f.readlines())
sourcecode = self.source[pp_file]
for line, text in enumerate(sourcecode.splitlines()):
m = self.insertPattern.match(text)
if m:
filename = m.group(1)
logger.info( "inserting code from {0}".format(filename) )
self.insertSource(filename)
else:
if self.codelinePattern.match(text):
self.sourcelines.append((text, line+1, pp_file))
labelPattern = re.compile('(\w+):\s+([^#\n\r]*)')
opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE)
varPattern = re.compile('var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}') #
def parse(self):
""" parse the code
"""
logger = logging.getLogger(__name__)
self.code = []
self.variabledict = collections.OrderedDict()
self.defines = dict()
addr_offset = 0
for text, lineno, sourcename in self.sourcelines:
m = self.varPattern.match(text)
if m:
self.addVariable(m, lineno, sourcename)
else:
m = self.definePattern.match(text)
if m:
self.addDefine(m, lineno, sourcename)
else:
# extract any JMP label, if present
m = self.labelPattern.match(text)
if m:
label, text = m.groups() #so the operation after ":" still gets parsed CWC 08162012
else:
label = None #The label for non-jump label line is NONE CWC 08172012
# search OPS list for a match to the current line
m = self.opPattern.match(text)
if m:
op, args = m.groups()
op = op.upper()
# split and remove whitespace
arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',')]
#substitute the defined variable directly with the corresponding value CWC 08172012
arglist = [ self.defines[x] if x in self.defines else x for x in arglist ]
#check for dds commands so CHAN commands can be inserted
if (op[:3] == 'DDS'):
try:
board = self.adIndexList[int(arglist[0])][0]
except ValueError:
raise ppexception("DDS argument does not resolve to integer", sourcename, lineno, arglist[0])
chan = self.adIndexList[int(arglist[0])][1]
if (self.adBoards[board].channelLimit != 1):
#boards with more than one channel require an extra channel selection command
chanData = self.adBoards[board].addCMD(chan)
chanData += (int(board) << 16)
self.code.append((len(self.code)+addr_offset, 'DDSCHN', chanData, label, sourcename, lineno))
data = arglist if len(arglist)>1 else arglist[0]
self.addLabel( label, len(self.code), sourcename, lineno)
self.code.append((len(self.code)+addr_offset, op, data, label, sourcename, lineno))
else:
logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno) )
raise ppexception("Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno),
sourcename, lineno, text)
self.appendVariableCode()
return self.code
def appendVariableCode(self):
""" append all variables to the instruction part of the code
"""
for var in list(self.variabledict.values()):
address = len(self.code)
self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0 ))
var.address = address
def addVariable(self, m, lineno, sourcename):
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug( "Variable {0} {1} {2}".format( m.groups(), lineno, sourcename ) )
var = Variable()
label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups()]
var.name = label
var.origin = sourcename
var.enabled = True
if var.encoding not in encodings:
raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, sourcename, lineno), sourcename, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(sourcename, data) )
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter( var.value, var.encoding )
else:
var.value = Q(float(data))
# var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :)
data = int(round(float(data)))
if label in self.defines:
logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(sourcename, label, data, self.defines[label]) )
raise ppexception("variable redifinition", sourcename, lineno, label)
else:
self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({ label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
# code is (address, operation, data, label or variablename, currentfile)
def toBytecode(self):
""" generate bytecode from code
"""
logger = logging.getLogger(__name__)
logger.debug( "\nCode ---> ByteCode:" )
self.bytecode = []
for line in self.code:
logger.debug( "{0}: {1}".format(hex(line[0]), line[1:] ))
bytedata = 0
if line[1] not in OPS:
raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1])
byteop = OPS[line[1]]
try:
data = line[2]
#attempt to locate commands with constant data
if (data == ''):
#found empty data
bytedata = 0
elif isinstance(data, (int, int)):
bytedata = data
elif isinstance(data, float):
bytedata = int(data)
elif isinstance(data, str): # now we are dealing with a variable and need its address
bytedata = self.variabledict[line[2]].address if line[2] in self.variabledict else self.labeldict[line[2]]
elif isinstance(data, list): # list is what we have for DDS, will have 8bit channel and 16bit address
channel, data = line[2]
if isinstance(data, str):
data = self.variabledict[data].address
bytedata = ((int(channel) & 0xf) << 16) | (int(data) & 0x0fff)
except KeyError:
logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n".format(line[4], data) )
raise ppexception("{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data)
self.bytecode.append((byteop, bytedata))
logger.debug( "---> {0} {1}".format(hex(byteop), hex(bytedata)) )
return self.bytecode
def convertParameter(self, mag, encoding=None ):
""" convert a dimensioned parameter to the binary value
expected by the hardware. The conversion is determined by the variable encoding
"""
if is_Q(mag):
if mag.dimensionality == Dimensions.time:
result = int(round(mag / self.timestep))
else:
step, unit, _, mask = encodings[encoding]
result = int(round(mag.m_as(unit) / step)) & mask
else:
if encoding:
step, unit, _, mask = encodings[encoding]
result = int(round(mag/step)) & mask
else:
result = mag
return result
def compileCode(self):
self.parse()
self.toBytecode()
def exitcode(self, code):
if code in self._exitcodes:
var = self._exitcodes[code]
if var.comment:
return var.comment
else:
return var.name
else:
return "Exitcode {0} Not found".format(code)
