def defineParsers():
#Enable a fast parsing mode with caching.
ParserElement.enablePackrat()
#end of line terminates statements, so it is not regular whitespace
ParserElement.setDefaultWhitespaceChars('\t ')
func_call = Forward() #forward declaration because this is a recursive rule
#The "terminal" rules
symbol = Word(alphas+'_-', alphanums+'_-') .setParseAction(action_symbol)
q_symbol = quotedString .setParseAction(action_q_symbol)
bracket_term = Literal("(").suppress() - func_call \
+ Literal(")").suppress()
word = symbol | q_symbol | bracket_term
#The function call
#Parse: "foo | bar | baz" or "foo"
pipeline = (word + ZeroOrMore("|" - word)) .setParseAction(action_pipeline)
#Parse "foo|bar op1 op2 op3"
func_call << (pipeline - ZeroOrMore(word)) .setParseAction(action_func_call)
#High level structure of program
line = LineEnd() | func_call - LineEnd() #empty line or function call
program = ZeroOrMore(line) + StringEnd() #multiple lines are a program
#define the comments
program.ignore('%' + restOfLine)
#no tab expansion
program.parseWithTabs()
#return additional func_call parser to make testing more easy
return program, func_call
def parse_enum_defs(s, parsers=None):
"""Parse all of the enum defs in a string, and add them to a list of type parsers
:Parameters:
- `s`: the string to parse
- `parsers`: the dictionary of parsers to which the new ones will be added (optional)
@return: a list of parsers for parsing type declarations
>>> test_string = \"""# A sample file
... a foo
... bar baz
...
... typedef enum { # a comment
... START, /* another comment */
... END
... } RUNMARK;
...
... typedef struct {
... float x;
... int y
... } GOO
...
... typedef enum {
... OAK,
... MAPLE
... } TREES;
...
... GOO 3.4 6
... \"""
>>> parsed_defs = parse_enum_defs(test_string)
>>> runmark_parser = parsed_defs['RUNMARK']
>>> runmark_parser.parseString("START")[0]
'START'
>>> runmark_parser.parseString("END")[0]
'END'
>>> try:
... runmark_parser.parseString("41")[0]
... except:
... print "parser threw an exception"
parser threw an exception
>>>
>>> tree_parser = parsed_defs['TREES']
>>> tree_parser.parseString("MAPLE")[0]
'MAPLE'
"""
type_parser = {} if parsers is None else parsers
one_enum_def_parser = make_one_enum_def_parser()
not_enum = ((value_name | struct_declaration_start | possible_type_name
| right_brace) + restOfLine).suppress()
enum_def_parser = ZeroOrMore(Group(one_enum_def_parser)
| not_enum) + stringEnd
enum_def_parser.ignore(hash_comment)
enum_def_parser.ignore(cStyleComment)
for enum in enum_def_parser.parseString(s):
type_parser[enum['enum_name']] = oneOf(enum['values'].asList())
return type_parser
def gramma():
## Tokens
point = Literal('.')
prefix_op = Literal('.')
choice_op = Literal('+')
parallel = Literal("||") | Literal("<>")
#ident = Word(alphas, alphanums+'_')
ratename = Word(alphas.lower(), alphanums + "_")
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
lsqpar = Literal('[').suppress()
rsqpar = Literal(']').suppress()
define = Literal('=')
semicol = Literal(';').suppress()
col = Literal(',').suppress()
number = Word(nums)
integer = number
floatnumber = Combine(integer + Optional(point + Optional(number)))
passiverate = Word('infty') | Word('T')
internalrate = Word('tau')
pound = Literal('#').suppress()
percent = Literal('%').suppress()
peparate = (ratename | floatnumber | internalrate
| passiverate).setParseAction(_check_var)
peparate_indef = floatnumber | internalrate | passiverate
sync = Word('<').suppress() + ratename + ZeroOrMore(col + ratename) + Word(
'>').suppress()
coop_op = (parallel | sync).setParseAction(_create_sync_set)
activity = (ratename + col + peparate).setParseAction(_create_activity)
procdef = (Word(alphas.upper(), alphanums + "_") +
Optional(lsqpar + peparate_indef +
rsqpar)).setParseAction(_create_procdef)
## RATES Definitions
ratedef = (Optional(percent) + ratename + define +
peparate_indef).setParseAction(assign_var) + semicol
prefix = Forward()
choice = Forward()
coop = Forward()
process = (activity
| procdef
| lpar + coop + rpar).setParseAction(_create_process)
prefix << (process +
ZeroOrMore(prefix_op + prefix)).setParseAction(_create_prefix)
choice << (prefix +
ZeroOrMore(choice_op + choice)).setParseAction(_create_choice)
coop << (choice + ZeroOrMore(coop_op + coop)).setParseAction(_create_coop)
rmdef = (Optional(pound) + procdef + define + coop +
semicol).setParseAction(_create_definition)
system_eq = Optional(pound) + coop
pepa = ZeroOrMore(ratedef) + ZeroOrMore(rmdef) + system_eq.setParseAction(
_create_system_equation)
pepacomment = '//' + restOfLine
pepa.ignore(pepacomment)
return pepa
def parse_header(s, enum_def_parser, struct_def_parser, struct_parsers):
"""Create a dictionary of keyword assignments in a yanny par file
:Parameters:
- `s`: the string to parse
- `enum_def_parser`: a parser that parses one enum definition
- `struct_def_parser`: a parser that parses on struct definition
- `struct_parsers`: a dictionary of structures that parse structure data
@return: a dictionary with the keyword assignments
>>> test_string = \"""# A sample file
... a foo
... bar baz goo # no more
...
... typedef enum {
... START,
... END
... } RUNMARK;
...
... typedef struct {
... float x;
... int y
... } GOO
...
... typedef enum {
... OAK,
... MAPLE
... } TREES;
...
... GOO 3.4 6
... \"""
>>>
>>> enum_def_parser = make_one_enum_def_parser()
>>> type_parsers = parse_enum_defs(test_string, base_type_parsers)
>>> struct_def_parser = make_one_struct_def_parser(type_parsers)
>>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers)
>>> h = parse_header(test_string, enum_def_parser, struct_def_parser, struct_parsers)
>>> print h['a']
foo
>>> print h['bar']
baz goo
"""
one_header_assignment_parser = make_one_header_assignment_parser(
struct_parsers)
not_header = Or([enum_def_parser, struct_def_parser] + \
[struct_parsers[k] for k in struct_parsers.keys()]).suppress()
header_parser = ZeroOrMore(not_header
| one_header_assignment_parser) + stringEnd
header_parser.ignore(hash_comment)
header_parser.ignore(cStyleComment)
header = {}
for d in header_parser.parseString(s):
header[d['name']] = d['value'].partition('#')[0].lstrip().rstrip()
return header
def _parse_items(self, source):
ParserElement.setDefaultWhitespaceChars(' \t\r')
EOL = LineEnd().suppress()
comment = Literal('#') + Optional( restOfLine ) + EOL
string = CharsNotIn("\n")
line = Group(
Word(alphanums + '-')('key') + Literal(':').suppress() + Optional(Combine(string + ZeroOrMore(EOL + Literal(' ') + string)))("value") + EOL
)
group = ZeroOrMore(line)
group.ignore(comment)
return group.parseString(source, True)
def parser():
rule = Forward()
body = OneOrMore(CharsNotIn('{};') + ';')
sel = CharsNotIn('{};')
rule <<= sel + Group( '{' + ZeroOrMore( rule | body ) + '}' )
rule.setParseAction( make_action(Rule) )
stylesheet = ZeroOrMore( rule )
stylesheet.ignore( cStyleComment )
return stylesheet
def create_grammar():
global arrows
global stereotypes
assert len(arrows) > 0
assert len(stereotypes) > 0
linechars = ''.join((c for c in printables if c not in '}\n')) + ' \t'
norbracket = ''.join((c for c in printables if c != ']')) + ' \t'
nogt = ''.join((c for c in printables if c != '>')) + ' \t'
norparen = ''.join((c for c in printables if c != ')')) + ' \t'
line = Word(linechars)
cls_body = Group(ZeroOrMore(line))
classkeyword = Keyword('class').setResultsName('type')
st_names = stereotypes.keys()
st = Literal(st_names[0])
for s in st_names[1:]:
st = st | Literal(s)
stereotype = Group(
Optional(Literal('<<').suppress() + st + Literal('>>').suppress()))
identifier_list = Word(alphas) + ZeroOrMore(
Literal(',').suppress() + Word(alphas))
baseclasses = Group(Optional(Literal(':').suppress() + identifier_list))
cls = Group(stereotype + classkeyword + Word(alphas) + baseclasses + \
Literal('{').suppress() + cls_body + Literal('}').suppress())
arrow_names = arrows.keys()
arrow_names.sort(lambda x, y: -cmp(len(x), len(y)))
arrow = Keyword(arrow_names[0])
for ar in arrow_names[1:]:
arrow = arrow | Keyword(ar)
relation_caption = Literal('(').suppress() + Word(norparen) + \
Literal(')').suppress()
quantifier = Literal('[').suppress() + Word(norbracket) + Literal(
']').suppress()
relation = Group(
Word(alphas) + Group(Optional(quantifier)) + \
arrow.setResultsName('type') + \
Word(alphas) + Group(Optional(quantifier)) + \
Group(Optional(relation_caption))
)
grammar = ZeroOrMore(cls | relation)
grammar.ignore(cStyleComment)
grammar.ignore("//" + restOfLine)
return grammar
def build_parser():
key = Word(alphanums).setResultsName('key')
value = restOfLine.setParseAction(lambda string, location, tokens: tokens[
0].strip()).setResultsName('value')
property_ = Group(key + Suppress(Literal('=')) + value)
properties = Group(OneOrMore(property_)).setResultsName('properties')
section_name = (Suppress('[') + OneOrMore(CharsNotIn(']')) +
Suppress(']')).setResultsName('section')
section = Group(section_name + properties)
ini_file = ZeroOrMore(section).setResultsName('sections')
ini_file.ignore(pythonStyleComment)
return ini_file
def build_parser():
key = Word(alphanums).setResultsName('key')
value = restOfLine.setParseAction(
lambda string, location, tokens: tokens[0].strip()
).setResultsName('value')
property_ = Group(key + Suppress(Literal('=')) + value)
properties = Group(OneOrMore(property_)).setResultsName('properties')
section_name = (Suppress('[') + OneOrMore(CharsNotIn(']')) +
Suppress(']')).setResultsName('section')
section = Group(section_name + properties)
ini_file = ZeroOrMore(section).setResultsName('sections')
ini_file.ignore(pythonStyleComment)
return ini_file
def create_grammar():
global arrows
global stereotypes
assert len(arrows) > 0
assert len(stereotypes) > 0
linechars = ''.join((c for c in printables if c not in '}\n')) + ' \t'
norbracket = ''.join((c for c in printables if c != ']')) + ' \t'
nogt = ''.join((c for c in printables if c != '>')) + ' \t'
norparen = ''.join((c for c in printables if c != ')')) + ' \t'
line = Word(linechars)
cls_body = Group(ZeroOrMore(line))
classkeyword = Keyword('class').setResultsName('type')
st_names = stereotypes.keys()
st = Literal(st_names[0])
for s in st_names[1:]:
st = st | Literal(s)
stereotype = Group(Optional(Literal('<<').suppress() + st + Literal('>>').suppress()))
identifier_list = Word(alphas) + ZeroOrMore(Literal(',').suppress() + Word(alphas))
baseclasses = Group(Optional(Literal(':').suppress() + identifier_list))
cls = Group(stereotype + classkeyword + Word(alphas) + baseclasses + \
Literal('{').suppress() + cls_body + Literal('}').suppress())
arrow_names = arrows.keys()
arrow_names.sort(lambda x,y: -cmp(len(x), len(y)))
arrow = Keyword(arrow_names[0])
for ar in arrow_names[1:]:
arrow = arrow | Keyword(ar)
relation_caption = Literal('(').suppress() + Word(norparen) + \
Literal(')').suppress()
quantifier = Literal('[').suppress() + Word(norbracket) + Literal(']').suppress()
relation = Group(
Word(alphas) + Group(Optional(quantifier)) + \
arrow.setResultsName('type') + \
Word(alphas) + Group(Optional(quantifier)) + \
Group(Optional(relation_caption))
)
grammar = ZeroOrMore(cls | relation)
grammar.ignore(cStyleComment)
grammar.ignore("//" + restOfLine)
return grammar
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def gramma():
## Tokens
point = Literal('.')
prefix_op = Literal('.')
choice_op = Literal('+')
parallel = Literal("||") | Literal("<>")
#ident = Word(alphas, alphanums+'_')
ratename = Word(alphas.lower(),alphanums+"_")
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
lsqpar = Literal('[').suppress()
rsqpar = Literal(']').suppress()
define = Literal('=')
semicol = Literal(';').suppress()
col = Literal(',').suppress()
number = Word(nums)
integer = number
floatnumber = Combine( integer + Optional( point + Optional(number)))
passiverate = Word('infty') | Word('T')
internalrate = Word('tau')
pound = Literal('#').suppress()
percent = Literal('%').suppress()
peparate = (ratename | floatnumber | internalrate | passiverate).setParseAction(_check_var)
peparate_indef = floatnumber | internalrate | passiverate
sync = Word('<').suppress() + ratename + ZeroOrMore(col + ratename) + Word('>').suppress()
coop_op = (parallel | sync).setParseAction(_create_sync_set)
activity = (ratename + col + peparate).setParseAction(_create_activity)
procdef = (Word(alphas.upper(), alphanums+"_") + Optional(lsqpar + peparate_indef + rsqpar)).setParseAction(_create_procdef)
## RATES Definitions
ratedef = (Optional(percent)+ratename + define + peparate_indef).setParseAction(assign_var) + semicol
prefix = Forward()
choice = Forward()
coop = Forward()
process = ( activity
| procdef
| lpar + coop + rpar
).setParseAction(_create_process)
prefix << (process + ZeroOrMore(prefix_op + prefix)).setParseAction( _create_prefix)
choice << (prefix + ZeroOrMore(choice_op + choice)).setParseAction(_create_choice)
coop << (choice + ZeroOrMore(coop_op + coop)).setParseAction(_create_coop)
rmdef = (Optional(pound) + procdef + define + coop + semicol).setParseAction(_create_definition)
system_eq = Optional(pound) + coop
pepa = ZeroOrMore(ratedef) + ZeroOrMore(rmdef) + system_eq.setParseAction(_create_system_equation)
pepacomment = '//' + restOfLine
pepa.ignore(pepacomment)
return pepa
def get_language():
"""Create or retrieve the parse tree for defining a sensor graph."""
global sensor_graph, statement
if sensor_graph is not None:
return sensor_graph
_create_primitives()
_create_simple_statements()
_create_block_bnf()
sensor_graph = ZeroOrMore(statement) + StringEnd()
sensor_graph.ignore(comment)
return sensor_graph
def _prepare_parser():
number = Regex(r'-?\d+')
local = Regex(r'%[A-Za-z0-9._]+')
glob = Regex(r'@[A-Za-z0-9._]+')
meta = Regex(r'![A-Za-z0-9._]+')
keywords = lambda keywords: MatchFirst(
Keyword(word) for word in keywords.split())
seplist = lambda entry: delimitedList(entry) | Empty()
label = local + ':'
unused_def = (keywords('target declare attributes') | '!') + restOfLine
type_ = Forward()
void = Keyword('void')
scalar_type = keywords('i1 i8 i16 i32 i64') | void
types_list = seplist(type_)
struct_type = '{' + types_list - '}'
array_type = '[' - number - 'x' - type_ - ']'
type_ << (scalar_type | local | struct_type | array_type)
type_def = local + '=' + Keyword('type') - struct_type
value = Forward()
typed_value = type_ + value
value_list = seplist(typed_value)
compound_value = '{' + value_list - '}'
array_value = '[' + value_list - ']'
kw_value = keywords('zeroinitializer null true false')
value << (number | kw_value | compound_value | array_value)
linkage = Optional(keywords('private external internal common'),
'external')
align = Optional(',' + Keyword('align') - number)
metas = seplist(meta + meta)
global_tag = keywords('global constant')
initializer = Optional(value, default='undef')
global_def = glob - '=' - linkage - global_tag - type_ - initializer - align - metas
definition = unused_def | type_def | global_def
llvm = ZeroOrMore(definition)
comment = ';' + restOfLine
llvm.ignore(comment)
return llvm
def getkw_bnf(self):
lcb = Literal("{").suppress()
rcb = Literal("}").suppress()
lsb = Literal("[").suppress()
rsb = Literal("]").suppress()
lps = Literal("(").suppress()
rps = Literal(")").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_sect = rcb
end_data = Literal('$end').suppress()
prtable = srange("[0-9a-zA-Z]") + '!$%&*+-./<>?@^_|~:'
kstr = Word(prtable) ^ quotedString.setParseAction(removeQuotes)
name = Word(alphas + "_", alphanums + "_")
vec=lsb+delimitedList(Word(prtable) ^ Literal("\n").suppress() ^\
quotedString.setParseAction(removeQuotes))+rsb
key = kstr ^ vec
keyword = name + eql + kstr
vector = name + eql + vec
data = Combine(dmark + name) + SkipTo(end_data) + end_data
data.setParseAction(self.store_data)
sect = name + lcb
sect.setParseAction(self.add_sect)
key_sect = name + Group(lps + kstr + rps) + lcb
key_sect.setParseAction(self.add_sect)
vec_sect = name + Group(lps + vec + rps) + lcb
vec_sect.setParseAction(self.add_vecsect)
end_sect.setParseAction(self.pop_sect)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
section = Forward()
input = section ^ data ^ keyword ^ vector
sectdef = sect ^ key_sect ^ vec_sect
section << sectdef + ZeroOrMore(input) + rcb
bnf = ZeroOrMore(input)
bnf.ignore(pythonStyleComment)
return bnf
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
arg_begin = Literal("(").suppress()
arg_end = Literal(")").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
# Helper definitions
kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin
vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | key_sect | vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
key_sect.setParseAction(self.add_sect)
vec_sect.setParseAction(self.add_vecsect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def _grammar(self):
ident = Word(alphanums + ".")
semi = Literal(";").suppress()
# lrb = Literal("(").suppress()
# rrb = Literal(")").suppress()
lcb = Literal("{").suppress()
rcb = Literal("}").suppress()
Value = SkipTo(semi)
KeyValue = Dict(Group(ident + Value + semi))
Dictionary = Forward()
Block = lcb + ZeroOrMore(Dictionary | KeyValue) + rcb
Dictionary << Dict(Group(ident + Block))
ParameterFile = ZeroOrMore(Dictionary | KeyValue)
ParameterFile.ignore(cStyleComment)
ParameterFile.ignore(cppStyleComment)
return ParameterFile
def _grammar(self):
ident = Word(alphanums + ".")
semi = Literal(";").suppress()
# lrb = Literal("(").suppress()
# rrb = Literal(")").suppress()
lcb = Literal("{").suppress()
rcb = Literal("}").suppress()
Value = SkipTo(semi)
KeyValue = Dict(Group(ident + Value + semi))
Dictionary = Forward()
Block = lcb + ZeroOrMore(Dictionary | KeyValue) + rcb
Dictionary << Dict(Group(ident + Block))
ParameterFile = ZeroOrMore(Dictionary | KeyValue)
ParameterFile.ignore(cStyleComment)
ParameterFile.ignore(cppStyleComment)
return ParameterFile
class VlogSource(VlogBase):
def __repr__(self):
return 'VlogSource'
@syntax_root('VlogSource')
def _source(self):
"""
source_text ::= { description }
description ::= module_declaration
| udp_declaration
| config_declaration
"""
self.module = VlogPModule()
self.source = ZeroOrMore(self.module.top)
self.source.ignore(cStyleComment)
self.source.ignore(cppStyleComment)
self.source.ignore(self.compiler_directive)
return self.source
@syntax_tree('VlogSource', priority=10, with_name='compiler_directive')
def _compiler_directive(self):
"""
Get rid of Compiler Directives
"""
# compiler directives
self._compiler_directive = Combine(
"`" + oneOf("define undef ifndef ifdef else endif default_nettype "
"include resetall timescale unconnected_drive "
"nounconnected_drive celldefine endcelldefine") +
restOfLine)
return self._compiler_directive
def find_procedures_headers(self):
CREATE = CaselessKeyword("CREATE")
OR = CaselessKeyword("OR")
REPLACE = CaselessKeyword("REPLACE")
FUNCTION = CaselessKeyword("FUNCTION")
IN = CaselessKeyword("IN")
OUT = CaselessKeyword("OUT")
INOUT = CaselessKeyword("INOUT")
VARIADIC = CaselessKeyword("VARIADIC")
NAME = (Word(alphas, alphanums + "_."))("name")
ALIAS = Word(alphas, alphanums + "_")
TYPE = (
Word(alphas, alphanums + "[]_. ", ) + Suppress(Optional(Literal("(") + Word(nums) + Literal(")")))
)
PRM = (
(Optional(IN | OUT | INOUT | VARIADIC | (OUT + VARIADIC)) +
Optional(ALIAS) +
TYPE) | TYPE
).setParseAction(lambda res: " ".join([w.strip() for w in res]))
COMMENT = "--" + restOfLine
COMMA = Suppress(",")
PARAMS = ZeroOrMore(
PRM +
Optional(COMMA)
)("input")
PARAMS.ignore(COMMENT)
HEADER = (
CREATE + Optional(OR) + Optional(REPLACE) + FUNCTION + NAME +
Suppress("(") + PARAMS + Suppress(")")
).setParseAction(lambda res: {"name": res.name, "input": res.input})
parse_header = OneOrMore(HEADER | Suppress(SkipTo(HEADER)))
parse_header.ignore(COMMENT)
parse_header.ignore(cStyleComment)
try:
headers = parse_header.parseString(self._sql)
except Exception as error:
print self._fpath
raise error
return headers
def get_parser(self):
declaration = Forward()
keyword = (Keyword("enum") | Keyword("case") | Keyword("struct")
| Keyword("default") | Keyword("switch") | Keyword("union")
| Keyword("const") | Keyword("unsigned") | Keyword("int")
| Keyword("hyper") | Keyword("float") | Keyword("double")
| Keyword("bool") | Keyword("typedef") | Keyword("opaque")
| Keyword("string") | Keyword("void") | Keyword("program")
| Keyword("version"))
identifier = NotAny(keyword) + Word(
alphas + alphas.upper(),
alphanums + alphanums.upper() + "_",
asKeyword=True)
constant = Combine(Optional("-") + Word(nums))
constant.setParseAction(lambda s, l, t: [int(t[0])])
value = constant | identifier
enum_body = Literal("{").suppress() + identifier + Literal(
"=").suppress() + value + ZeroOrMore(
Literal(",").suppress() + identifier +
Literal("=").suppress() + value) + Literal("}").suppress()
enum_type_spec = Literal("enum").suppress() + enum_body
enum_body.setParseAction(self.parse_enum)
struct_body = Literal("{").suppress() + OneOrMore(
declaration + Literal(";").suppress()) + Literal("}").suppress()
struct_type_spec = Literal("struct").suppress() + struct_body
struct_body.setParseAction(self.parse_struct)
case_stmt = Literal("case").suppress() + value + Literal(
":").suppress() + declaration + Literal(";").suppress()
default_stmt = Literal("default") + Literal(
":").suppress() + declaration + Literal(";").suppress()
union_body = Literal("switch").suppress() + Literal("(").suppress(
) + declaration + Literal(")").suppress() + Literal("{").suppress(
) + Group(OneOrMore(Group(case_stmt)) +
Optional(Group(default_stmt))) + Literal("}").suppress()
union_type_spec = Literal("union").suppress() + union_body
union_body.setParseAction(self.parse_union)
constant_def = Literal("const").suppress() + identifier + Literal(
"=").suppress() + constant + Literal(";").suppress()
constant_def.setParseAction(self.parse_const)
type_spec = ((Optional(Literal("unsigned")) +
Literal("int")).setParseAction(self.parse_builtin) |
(Optional(Literal("unsigned")) +
Literal("hyper")).setParseAction(self.parse_builtin)
| Literal("float").setParseAction(self.parse_builtin)
| Literal("double").setParseAction(self.parse_builtin)
| Literal("bool").setParseAction(self.parse_builtin)
| enum_type_spec | struct_type_spec | union_type_spec
| identifier)
proc_return = Literal("void") | type_spec
procedure_def = proc_return + identifier + Literal("(").suppress() + (
Literal("void") | type_spec) + ZeroOrMore(
Literal(",").suppress() +
type_spec) + Literal(")").suppress() + Literal(
"=").suppress() + constant + Literal(";").suppress()
procedure_def.setParseAction(self.parse_procedure_def)
version_def = Literal("version").suppress() + identifier + Literal(
"{").suppress() + OneOrMore(procedure_def) + Literal("}").suppress(
) + Literal("=").suppress() + constant + Literal(";").suppress()
version_def.setParseAction(self.parse_version_def)
program_body = Literal("{").suppress() + Group(
OneOrMore(version_def)) + Literal("}").suppress()
type_def = (
(Literal("typedef") + declaration + Literal(";")) |
(Literal("enum") + identifier + enum_body + Literal(";")) |
(Literal("struct") + identifier + struct_body + Literal(";")) |
(Literal("union") + identifier + union_body + Literal(";")) |
(Literal("program") + identifier + program_body +
Literal("=").suppress() + constant + Literal(";")))
type_def.setParseAction(self.parse_type_def)
declaration << (
(type_spec + identifier + Literal("[") + value + Literal("]")) |
(type_spec + identifier + Literal("<") + value + Literal(">")) |
(type_spec + identifier) | (Literal("opaque") + identifier +
Literal("[") + value + Literal("]")) |
(Literal("opaque") + identifier + Literal("<") + value +
Literal(">")) | (Literal("string") + identifier + Literal("<") +
value + Literal(">")) |
(type_spec + Literal("*") + identifier) | Literal("void"))
declaration.setParseAction(self.parse_decl)
definition = type_def | constant_def
specification = ZeroOrMore(definition)
comment = (Literal("#") + restOfLine).suppress()
specification.ignore(comment)
return specification
class pppCompiler:
def __init__(self):
self.initBNF()
self.symbols = SymbolTable()
def initBNF(self):
indentStack = [1]
encoding = Literal("<").suppress() + identifier("encoding") + Literal(
">").suppress()
constdecl = Group((const + identifier + assign + value).setParseAction(
self.const_action))
vardecl = Group(
(type_("type_") + Optional(encoding) + identifier("name") +
Optional(assign + value("value") + Optional(identifier)("unit"))
).setParseAction(self.var_action))
insertdecl = Group(
(insert + dblQuotedString + LineEnd().suppress()).setParseAction(
self.insert_action))
procedurecall = Group((identifier + Literal("(").suppress() + Optional(
delimitedList(
(identifier + Optional(assign + identifier)).setParseAction(
self.named_param_action))) +
Literal(")").suppress()).setParseAction(
self.procedurecall_action))
condition = Group(
(identifier("leftidentifier") + comparison("comparison") +
(identifier("identifier")
| value.setParseAction(self.value_action))).setParseAction(
self.condition_action))("condition")
pointer = Literal("*") + identifier
rExp = Forward()
#numexpression = Forward()
opexpression = (identifier("operand") +
(Literal(">>") | Literal("<<") | Literal("+")
| Literal("*") | Literal("/") | Literal("-"))("op") +
Group(rExp)("argument")).setParseAction(
self.opexpression_action)
rExp << (
procedurecall | opexpression | identifier("identifier")
| value.setParseAction(self.value_action) |
#Group( Suppress("(") + rExp + Suppress(")") ) |
#Group( "+" + rExp) |
#Group( "-" + rExp) |
Group(Literal("not") + rExp))
rExpCondition = Group(
(Optional(not_)("not_") + rExp("rExp"))).setParseAction(
self.rExp_condition_action)("condition")
rExp.setParseAction(self.rExp_action)
assignment = ((identifier | pointer)("lval") + assign +
rExp("rval")).setParseAction(self.assignment_action)
addassignment = ((identifier | pointer)("lval") +
(Literal("+=") | Literal("-=") | Literal("*=")
| Literal("&=") | Literal("|=") | Literal(">>=")
| Literal("/=") | Literal("<<="))("op") +
Group(rExp)("rval")).setParseAction(
self.addassignement_action)
statement = Forward()
statementBlock = indentedBlock(statement, indentStack).setParseAction(
self.statementBlock_action)
procedure_statement = Group(
(Keyword("def").suppress() + identifier("funcname") +
Literal("(").suppress() + Literal(")").suppress() +
colon.suppress() + statementBlock).setParseAction(
self.def_action))
while_statement = Group(
(Keyword("while").suppress() +
(condition | rExpCondition)("condition") + colon.suppress() +
statementBlock("statementBlock")).setParseAction(
self.while_action))
if_statement = (Keyword("if") + condition + colon +
statementBlock("ifblock") + Optional(
Keyword("else").suppress() + colon +
statementBlock("elseblock"))).setParseAction(
self.if_action)
statement << (procedure_statement | while_statement | if_statement
| procedurecall | assignment | addassignment)
decl = constdecl | vardecl | insertdecl | Group(statement)
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def assignment_action(self, text, loc, arg):
logging.getLogger(__name__).debug("assignment_action {0} {1}".format(
lineno(loc, text), arg))
try:
code = [
"# line {0} assignment {1}".format(lineno(loc, text),
line(loc, text))
]
rval_code = find_and_get(arg.rval, 'code')
if rval_code is not None:
code += arg.rval.code
elif arg.rval == "*P":
code.append(" LDWI")
elif 'identifier' in arg:
self.symbols.getVar(arg.identifier)
code.append(" LDWR {0}".format(arg.identifier))
if arg.lval == "*P":
code.append(" STWI")
elif arg.lval != "W":
symbol = self.symbols.getVar(arg.lval)
code.append(" STWR {0}".format(symbol.name))
if 'code' in arg:
arg['code'].extend(code)
else:
arg['code'] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def addassignement_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"addassignement_action {0} {1}".format(lineno(loc, text), arg))
try:
code = [
"# line {0}: add_assignment: {1}".format(
lineno(loc, text), line(loc, text))
]
if arg.rval[0] == '1' and arg.op in ['+=', '-=']:
self.symbols.getVar(arg.lval)
if arg.op == "+=":
code.append(" INC {0}".format(arg.lval))
else:
code.append(" DEC {0}".format(arg.lval))
else:
if 'code' in arg.rval:
code += arg.rval.code
self.symbols.getVar(arg.lval)
if arg.op == "-=":
raise CompileException(
"-= with expression needs to be fixed in the compiler"
)
code.append(" {0} {1}".format(opassignmentLookup[arg.op],
arg.lval))
elif 'identifier' in arg.rval:
self.symbols.getVar(arg.rval.identifier)
code.append(" LDWR {0}".format(arg.lval))
self.symbols.getVar(arg.lval)
code.append(" {0} {1}".format(opassignmentLookup[arg.op],
arg.rval.identifier))
code.append(" STWR {0}".format(arg.lval))
arg['code'] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def condition_action(self, text, loc, arg):
logging.getLogger(__name__).debug("condition_action {0} {1}".format(
lineno(loc, text), arg))
try:
code = [
"# line {0} condition {1}".format(lineno(loc, text),
line(loc, text))
]
if arg.leftidentifier != "W":
self.symbols.getVar(arg.leftidentifier)
code.append(' LDWR {0}'.format(arg.leftidentifier))
if arg.identifier == 'NULL' and arg.comparison in jmpNullCommands:
arg['jmpcmd'] = jmpNullCommands[arg.comparison]
else:
code.append(' {0} {1}'.format(
comparisonCommands[arg.comparison], arg.identifier))
arg["code"] = code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def rExp_condition_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"rExp_condition_action {0} {1}".format(lineno(loc, text), arg))
try:
code = [
"# line {0} rExp_condition {1}".format(lineno(loc, text),
line(loc, text))
]
condition_code = arg.condition.rExp['code']
if isinstance(condition_code, str):
if 'not_' in arg['condition']:
code += [" CMPEQUAL NULL"]
else:
code += [" CMPNOTEQUAL NULL"]
arg['code'] = code
else:
if 'not_' in arg['condition']:
arg['code'] = {
False: condition_code[True],
True: condition_code[False]
}
else:
arg['code'] = condition_code
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def named_param_action(self, text, loc, arg):
if len(arg) == 2:
arg[arg[0]] = arg[1]
return arg
def value_action(self, text, loc, arg):
if arg[0][0:2] == '0x':
value = int(arg[0], 16)
else:
value = int(arg[0])
arg["identifier"] = self.symbols.getInlineParameter("inlinevar", value)
return arg
def opexpression_action(self, text, loc, arg):
try:
logging.getLogger(__name__).debug(
"opexpression_action {0} {1}".format(lineno(loc, text), arg))
code = [
"# line {0}: shiftexpression {1}".format(
lineno(loc, text),
line(loc, text)), " LDWR {0}".format(arg.operand),
" {0} {1}".format(shiftLookup[arg.op],
arg.argument.identifier)
]
arg['code'] = code
logging.getLogger(__name__).debug(
"shiftexpression generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def procedurecall_action(self, text, loc, arg):
try:
logging.getLogger(__name__).debug(
"procedurecall_action {0} {1}".format(lineno(loc, text), arg))
procedure = self.symbols.getProcedure(arg[0])
code = [
"# line {0}: procedurecall {1}".format(lineno(loc, text),
line(loc, text))
]
opcode = procedure.codegen(self.symbols,
arg=arg.asList(),
kwarg=arg.asDict())
if isinstance(opcode, list):
code += opcode
else:
code = opcode
arg['code'] = code
logging.getLogger(__name__).debug(
"procedurecall generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def rExp_action(self, text, loc, arg):
logging.getLogger(__name__).debug("rExp_action {0} {1}".format(
lineno(loc, text), arg))
pass
def if_action(self, text, loc, arg):
logging.getLogger(__name__).debug("if_action {0} {1}".format(
lineno(loc, text), arg))
try:
block0 = [
"# line {0} if statement {1}".format(lineno(loc, text),
line(loc, text))
]
if isinstance(arg.condition.code, list):
block0 += arg.condition.code
JMPCMD = arg.condition.get('jmpcmd', {False: "JMPNCMP"})[False]
else:
JMPCMD = arg.condition.code[True]
if 'elseblock' in arg:
block1 = arg.ifblock.ifblock.code
block2 = arg.elseblock.elseblock[
'code'] if 'elseblock' in arg.elseblock else arg.elseblock[
'code']
else:
block1 = arg.ifblock.ifblock['code']
block2 = None
arg['code'] = [
IfGenerator(self.symbols, JMPCMD, block0, block1, block2)
]
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def while_action(self, text, loc, arg):
logging.getLogger(__name__).debug("while_action {0} {1}".format(
lineno(loc, text), arg))
try:
block0 = [
"# line {0} while_statement {1}".format(
lineno(loc, text), line(loc, text))
]
if 'code' in arg.condition:
if isinstance(arg.condition.code, list):
block1 = arg.condition.code
JMPCMD = arg.condition.get('jmpcmd',
{False: "JMPNCMP"})[False]
else:
JMPCMD = arg.condition.code[True]
block1 = []
elif 'rExp' in arg.condition and 'code' in arg.condition.rExp:
if isinstance(arg.condition.rExp.code, list):
block1 = arg.condition.rExp.code
JMPCMD = arg.condition.rExp.get('jmpcmd', "JMPNCMP")
else:
JMPCMD = arg.condition.rExp.code[True]
block1 = []
block2 = arg.statementBlock.statementBlock['code']
arg['code'] = [
WhileGenerator(self.symbols, JMPCMD, block0, block1, block2)
]
logging.getLogger(__name__).debug("while_action generated code ")
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def statementBlock_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"statementBlock_action {0} {1} {2}".format(lineno(loc, text),
arg.funcname, arg))
try:
code = list()
for command in arg[0]:
if 'code' in command:
code += command['code']
elif 'code' in command[0]:
code += command[0]['code']
arg[0]['code'] = code
logging.getLogger(__name__).debug(
"statementBlock generated code {0}".format(code))
except Exception as e:
raise CompileException(text, loc, str(e), self)
return arg
def def_action(self, text, loc, arg):
logging.getLogger(__name__).debug("def_action {0} {1} {2}".format(
lineno(loc, text), arg.funcname, arg))
try:
name = arg[0]
self.symbols.checkAvailable(name)
self.symbols[name] = FunctionSymbol(name, arg[1]['code'])
except Exception as e:
raise CompileException(text, loc, str(e), self)
def const_action(self, text, loc, arg):
try:
name, value = arg
logging.getLogger(__name__).debug(
"const_action {0} {1} {2} {3}".format(self.currentFile,
lineno(loc, text), name,
value))
self.symbols[name] = ConstSymbol(name, value)
except Exception as e:
raise CompileException(text, loc, str(e), self)
def var_action(self, text, loc, arg):
logging.getLogger(__name__).debug(
"var_action {0} {1} {2} {3} {4} {5} {6}".format(
self.currentFile, lineno(loc, text), arg["type_"],
arg.get("encoding"), arg["name"], arg.get("value"),
arg.get("unit")))
try:
type_ = arg["type_"] if arg["type_"] != "var" else None
self.symbols[arg["name"]] = VarSymbol(type_=type_,
name=arg["name"],
value=arg.get("value"),
encoding=arg.get("encoding"),
unit=arg.get("unit"))
except Exception as e:
raise CompileException(text, loc, str(e), self)
def insert_action(self, text, loc, arg):
try:
oldfile = self.currentFile
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile(self.currentFile)
self.currentFile = oldfile
except Exception as e:
raise CompileException(text, loc, str(e), self)
return result
def compileFile(self, filename):
self.currentFile = filename
result = self.program.parseFile(self.currentFile, parseAll=True)
allcode = list()
for element in result:
if not isinstance(element, str) and 'code' in element:
allcode += element['code']
elif not isinstance(element[0], str) and 'code' in element[0]:
allcode += element[0]['code']
header = self.createHeader()
codetext = "\n".join(header + allcode)
return codetext
def compileString(self, programText):
self.programText = programText
self.currentFile = "Memory"
result = self.program.parseString(self.programText, parseAll=True)
allcode = list()
for element in result:
if not isinstance(element, str) and 'code' in element:
allcode += element['code']
elif not isinstance(element[0], str) and 'code' in element[0]:
allcode += element[0]['code']
header = self.createHeader()
codetext = """# autogenerated
# DO NOT EDIT DIRECTLY
# The file will be overwritten by the compiler
#
"""
codetext += "\n".join(header + list(generate(allcode)))
self.reverseLineLookup = self.generateReverseLineLookup(codetext)
return codetext
def generateReverseLineLookup(self, codetext):
lookup = dict()
sourceline = None
for codeline, line in enumerate(codetext.splitlines()):
m = re.search('^\# line (\d+).*$', line)
if m:
sourceline = int(m.group(1))
else:
lookup[codeline + 1] = sourceline
return lookup
def createHeader(self):
header = ["# const values"]
for constval in self.symbols.getAllConst():
header.append("const {0} {1}".format(constval.name,
constval.value))
header.append("# variables ")
for var in self.symbols.getAllVar():
if var.type_ == "masked_shutter":
header.append("var {0} {1}, {2}".format(
var.name + "_mask",
var.value if var.value is not None else 0, "mask"))
header.append("var {0} {1}, {2}".format(
var.name, var.value if var.value is not None else 0,
"shutter {0}_mask".format(var.name)))
else:
optionals = [
s if s is not None else ""
for s in list_rtrim([var.type_, var.unit, var.encoding])
]
varline = "var {0} {1}".format(
var.name, var.value if var.value is not None else 0)
if len(optionals) > 0:
varline += ", " + ", ".join(optionals)
header.append(varline)
header.append("# inline variables")
# for value, name in self.symbols.inlineParameterValues.items():
# header.append("var {0} {1}".format(name, value))
header.append("# end header")
header.append("")
return header
identifier.setResultsName("tablename") + fromToken + identifier.setResultsName("filename")
#orderByClause = orderByToken +
selectStatement << (selectToken + columnNameList.setResultsName("columns") +
fromToken + identifier.setResultsName("tablename") +
Optional(whereClause) + Optional(orderByClause) +
Optional(limitClause))
BQLStatement = (selectStatement | createBtableStatement) + Optional(';')
BQL = ZeroOrMore(BQLStatement)
## allows comments
dashComment = "--" + restOfLine
BQL.ignore(dashComment)
def test(str):
print str, "->"
try:
tokens = BQL.parseString(str)
print "tokens = ", tokens
print "tokens.tablename =", tokens.tablename
print "tokens.filename =", tokens.filename
#print "tokens.where =", tokens.where
except ParseException, err:
print " " * err.loc + "^\n" + err.msg
print err
print
class PulseProgram:
""" Encapsulates a PulseProgrammer Program
loadSource( filename ) loads the contents of the file
The code is compiled in the following steps
parse() generates self.code
toBytecode() generates self.bytecode
toBinary() generates self.binarycode
the procedure updateVariables( dictionary ) updates variable values in the bytecode
"""
def __init__(self):
self.variabledict = collections.OrderedDict(
) # keeps information on all variables to easily change them later
self.labeldict = dict() # keep information on all labels
self.source = collections.OrderedDict(
) # dictionary of source code files (stored as strings)
self.code = [] # this is a list of lines
self.bytecode = [] # list of op, argument tuples
self.binarycode = bytearray() # binarycode to be uploaded
self._exitcodes = dict(
) # generate a reverse dictionary of variables of type exitcode
self.constDict = dict()
class Board:
channelLimit = 1
halfClockLimit = 500000000
self.adIndexList = [(x, 0) for x in range(6)]
self.adBoards = [Board()] * 6
self.timestep = Q(20.0, 'ns')
self.initBNF()
def initBNF(self):
constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action)
vardecl = (VAR + NAME + VALUE +
Optional(COMMA + Regex("[^#\n]*"))).setParseAction(
self.var_action)
insertdecl = (INSERT + dblQuotedString +
LineEnd().suppress()).setParseAction(self.insert_action)
LABEL = IDENTIFIER + COLON
COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") +
Regex("[^#\n]*").setWhitespaceChars(" \t") +
LineEnd().suppress())
COMMAND = COMMANDEXP.setParseAction(self.command_action)
LABELEDCOMMAND = (LABEL + COMMANDEXP).setParseAction(
self.label_command_action)
decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def const_action(self, text, loc, arg):
""" add the const to the self.constDict dictionary
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} const {2}".format(self.currentFile,
lineno(loc, text), arg))
label, value = arg
if label in self.constDict:
logger.error(
"Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'"
.format(self.currentFile, label, value, self.constDict[label]))
raise ppexception("Redefining variable", self.currentFile, lineno,
label)
else:
self.constDict[label] = int(value)
def var_action(self, text, loc, arg):
print("var_action", self.currentFile, lineno(loc, text), arg[0:2],
arg[2].split(",") if len(arg) > 2 else "")
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} Variable {2}".format(self.currentFile,
lineno(loc, text), arg))
var = Variable()
label, data = arg[:2]
fields = arg[2].split(",") if len(arg) > 2 else [None] * 3
fields += [None] * (3 - len(fields))
var.type, unit, var.encoding = [
x if x is None or '' else x.strip() for x in fields
]
var.name = label
var.origin = self.currentFile
var.enabled = True
if var.encoding not in encodings:
raise ppexception(
"unknown encoding {0} in file '{1}':{2}".format(
var.encoding, self.currentFile, lineno(loc, text)),
self.currentFile, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception(
"Evaluation error in file '{0}' on line: '{1}'".format(
self.currentFile, data))
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter(var.value, var.encoding)
else:
var.value = Q(float(data))
data = int(round(float(data)))
if label in self.defines:
logger.error(
"Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement."
% (self.currentFile, label, data, self.defines[label]))
raise ppexception("variable redifinition", self.currentFile,
lineno, label)
else:
self.defines[
label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
def command_action(self, text, loc, arg):
print("command_action", self.currentFile, lineno(loc, text), arg[0:1],
arg[1].split(",") if len(arg) > 1 else "")
def label_command_action(self, text, loc, arg):
print("label_command_action", self.currentFile, lineno(loc, text),
arg[0:2], arg[2].split(",") if len(arg) > 2 else "")
def addLabel(self, label, address, sourcename, lineno):
if label is not None:
self.labeldict[label] = address
def insert_action(self, text, loc, arg):
oldfile = self.currentFile
print("insert_action", lineno(loc, text), arg)
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile(self.currentFile)
self.currentFile = oldfile
print(result)
return result
def assembleFile(self, filename):
self.currentFile = filename
result = self.program.parseFile(self.currentFile)
return result
def setHardware(self, adIndexList, adBoards, timestep):
self.adIndexList = adIndexList
self.adBoards = adBoards
self.timestep = timestep
assert self.timestep.has_dimension('s')
def saveSource(self):
for name, text in self.source.items():
with open(os.path.join(self.pp_dir, name), 'w') as f:
f.write(text)
def loadSource(self, pp_file):
""" Load the source pp_file
#include files are loaded recursively
all code lines are added to self.sourcelines
for each source file the contents are added to the dictionary self.source
"""
self.source.clear()
self.pp_dir, self.pp_filename = os.path.split(pp_file)
self.sourcelines = []
self.insertSource(self.pp_filename)
self.compileCode()
def updateVariables(self, variables):
""" update the variable values in the bytecode
"""
logger = logging.getLogger(__name__)
for name, value in variables.items():
if name in self.variabledict:
var = self.variabledict[name]
address = var.address
var.value = value
logger.debug(
"updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}"
.format(name, value, address, int(var.data)))
var.data = self.convertParameter(value, var.encoding)
self.bytecode[address] = (self.bytecode[address][0], var.data)
self.variabledict[name] = var
else:
logger.error(
"variable {0} not found in dictionary.".format(name))
return self.bytecode
def variables(self):
mydict = dict()
for name, var in self.variabledict.items():
mydict.update({name: var.value})
return mydict
def variable(self, variablename):
return self.variabledict.get(variablename).value
def variableUpdateCode(self, variablename, value):
"""returns the code to update the variable directly on the fpga
consists of variablelocation and variablevalue
"""
var = self.variabledict[variablename]
data = self.convertParameter(value, var.encoding)
return bytearray(struct.pack('II', (var.address, data)))
def flattenList(self, l):
return [item for sublist in l for item in sublist]
def variableScanCode(self, variablename, values):
var = self.variabledict[variablename]
# [item for sublist in l for item in sublist] idiom for flattening of list
return self.flattenList([(var.address,
self.convertParameter(x, var.encoding))
for x in values])
def loadFromMemory(self):
"""Similar to loadSource
only this routine loads from self.source
"""
self.sourcelines = []
self._exitcodes = dict()
self.insertSource(self.pp_filename)
self.compileCode()
def toBinary(self):
""" convert bytecode to binary
"""
logger = logging.getLogger(__name__)
self.binarycode = bytearray()
for wordno, (op, arg) in enumerate(self.bytecode):
logger.debug("{0} {1} {2} {3}".format(
hex(wordno), hex(int(op)), hex(int(arg)),
hex(int((int(op) << 24) + int(arg)))))
self.binarycode += struct.pack('I', int((op << 24) + arg))
return self.binarycode
def currentVariablesText(self):
lines = list()
for name, var in iter(sorted(self.variabledict.items())):
lines.append("{0} {1}".format(name, var.value))
return '\n'.join(lines)
# routines below here should not be needed by the user
insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE)
codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE)
def insertSource(self, pp_file):
""" read a source file pp_file
calls itself recursively to for #insert
adds the contents of this file to the dictionary self.source
"""
logger = logging.getLogger(__name__)
if pp_file not in self.source:
with open(os.path.join(self.pp_dir, pp_file)) as f:
self.source[pp_file] = ''.join(f.readlines())
sourcecode = self.source[pp_file]
for line, text in enumerate(sourcecode.splitlines()):
m = self.insertPattern.match(text)
if m:
filename = m.group(1)
logger.info("inserting code from {0}".format(filename))
self.insertSource(filename)
else:
if self.codelinePattern.match(text):
self.sourcelines.append((text, line + 1, pp_file))
labelPattern = re.compile('(\w+):\s+([^#\n\r]*)')
opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE)
varPattern = re.compile(
'var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}'
) #
def parse(self):
""" parse the code
"""
logger = logging.getLogger(__name__)
self.code = []
self.variabledict = collections.OrderedDict()
self.defines = dict()
addr_offset = 0
for text, lineno, sourcename in self.sourcelines:
m = self.varPattern.match(text)
if m:
self.addVariable(m, lineno, sourcename)
else:
m = self.definePattern.match(text)
if m:
self.addDefine(m, lineno, sourcename)
else:
# extract any JMP label, if present
m = self.labelPattern.match(text)
if m:
label, text = m.groups(
) #so the operation after ":" still gets parsed CWC 08162012
else:
label = None #The label for non-jump label line is NONE CWC 08172012
# search OPS list for a match to the current line
m = self.opPattern.match(text)
if m:
op, args = m.groups()
op = op.upper()
# split and remove whitespace
arglist = [0] if args is None else [
0 if x is None else x.strip()
for x in args.split(',')
]
#substitute the defined variable directly with the corresponding value CWC 08172012
arglist = [
self.defines[x] if x in self.defines else x
for x in arglist
]
#check for dds commands so CHAN commands can be inserted
if (op[:3] == 'DDS'):
try:
board = self.adIndexList[int(arglist[0])][0]
except ValueError:
raise ppexception(
"DDS argument does not resolve to integer",
sourcename, lineno, arglist[0])
chan = self.adIndexList[int(arglist[0])][1]
if (self.adBoards[board].channelLimit != 1):
#boards with more than one channel require an extra channel selection command
chanData = self.adBoards[board].addCMD(chan)
chanData += (int(board) << 16)
self.code.append(
(len(self.code) + addr_offset, 'DDSCHN',
chanData, label, sourcename, lineno))
data = arglist if len(arglist) > 1 else arglist[0]
self.addLabel(label, len(self.code), sourcename,
lineno)
self.code.append((len(self.code) + addr_offset, op,
data, label, sourcename, lineno))
else:
logger.error(
"Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)"
.format(text, sourcename, lineno))
raise ppexception(
"Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)"
.format(text, sourcename,
lineno), sourcename, lineno, text)
self.appendVariableCode()
return self.code
def appendVariableCode(self):
""" append all variables to the instruction part of the code
"""
for var in list(self.variabledict.values()):
address = len(self.code)
self.code.append((address, 'NOP', var.data if var.enabled else 0,
None, var.origin, 0))
var.address = address
def addVariable(self, m, lineno, sourcename):
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug("Variable {0} {1} {2}".format(m.groups(), lineno,
sourcename))
var = Variable()
label, data, var.type, unit, var.encoding, var.comment = [
x if x is None else x.strip() for x in m.groups()
]
var.name = label
var.origin = sourcename
var.enabled = True
if var.encoding not in encodings:
raise ppexception(
"unknown encoding {0} in file '{1}':{2}".format(
var.encoding, sourcename, lineno), sourcename, lineno,
var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception(
"Evaluation error in file '{0}' on line: '{1}'".format(
sourcename, data))
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter(var.value, var.encoding)
else:
var.value = Q(float(data))
# var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :)
data = int(round(float(data)))
if label in self.defines:
logger.error(
"Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement."
% (sourcename, label, data, self.defines[label]))
raise ppexception("variable redifinition", sourcename, lineno,
label)
else:
self.defines[
label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
# code is (address, operation, data, label or variablename, currentfile)
def toBytecode(self):
""" generate bytecode from code
"""
logger = logging.getLogger(__name__)
logger.debug("\nCode ---> ByteCode:")
self.bytecode = []
for line in self.code:
logger.debug("{0}: {1}".format(hex(line[0]), line[1:]))
bytedata = 0
if line[1] not in OPS:
raise ppexception("Unknown command {0}".format(line[1]),
line[4], line[5], line[1])
byteop = OPS[line[1]]
try:
data = line[2]
#attempt to locate commands with constant data
if (data == ''):
#found empty data
bytedata = 0
elif isinstance(data, (int, int)):
bytedata = data
elif isinstance(data, float):
bytedata = int(data)
elif isinstance(
data, str
): # now we are dealing with a variable and need its address
bytedata = self.variabledict[line[2]].address if line[
2] in self.variabledict else self.labeldict[line[2]]
elif isinstance(
data, list
): # list is what we have for DDS, will have 8bit channel and 16bit address
channel, data = line[2]
if isinstance(data, str):
data = self.variabledict[data].address
bytedata = (
(int(channel) & 0xf) << 16) | (int(data) & 0x0fff)
except KeyError:
logger.error(
"Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n"
.format(line[4], data))
raise ppexception(
"{0}: Unknown variable {1}".format(line[4], data), line[4],
line[5], data)
self.bytecode.append((byteop, bytedata))
logger.debug("---> {0} {1}".format(hex(byteop), hex(bytedata)))
return self.bytecode
def convertParameter(self, mag, encoding=None):
""" convert a dimensioned parameter to the binary value
expected by the hardware. The conversion is determined by the variable encoding
"""
if is_Q(mag):
if mag.dimensionality == Dimensions.time:
result = int(round(mag / self.timestep))
else:
step, unit, _, mask = encodings[encoding]
result = int(round(mag.m_as(unit) / step)) & mask
else:
if encoding:
step, unit, _, mask = encodings[encoding]
result = int(round(mag / step)) & mask
else:
result = mag
return result
def compileCode(self):
self.parse()
self.toBytecode()
def exitcode(self, code):
if code in self._exitcodes:
var = self._exitcodes[code]
if var.comment:
return var.comment
else:
return var.name
else:
return "Exitcode {0} Not found".format(code)
def SPICE_BNF():
global bnf
if not bnf:
# punctuation
colon = Literal(":").suppress()
lbrace = Literal("{").suppress()
rbrace = Literal("}").suppress()
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
lparen = Literal("(").suppress()
rparen = Literal(")").suppress()
equals = Literal("=").suppress()
comma = Literal(",").suppress()
semi = Literal(";").suppress()
# primitive types
int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8))
uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8))
int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16))
uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16))
int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32))
uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32))
int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64))
uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64))
# keywords
channel_ = Keyword("channel")
enum32_ = Keyword("enum32").setParseAction(replaceWith(32))
enum16_ = Keyword("enum16").setParseAction(replaceWith(16))
enum8_ = Keyword("enum8").setParseAction(replaceWith(8))
flags32_ = Keyword("flags32").setParseAction(replaceWith(32))
flags16_ = Keyword("flags16").setParseAction(replaceWith(16))
flags8_ = Keyword("flags8").setParseAction(replaceWith(8))
channel_ = Keyword("channel")
server_ = Keyword("server")
client_ = Keyword("client")
protocol_ = Keyword("protocol")
typedef_ = Keyword("typedef")
struct_ = Keyword("struct")
message_ = Keyword("message")
image_size_ = Keyword("image_size")
bytes_ = Keyword("bytes")
cstring_ = Keyword("cstring")
switch_ = Keyword("switch")
default_ = Keyword("default")
case_ = Keyword("case")
identifier = Word(alphas, alphanums + "_")
enumname = Word(alphanums + "_")
integer = (
Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF"))
| Word(nums + "+-", nums)).setName("int").setParseAction(cvtInt)
typename = identifier.copy().setParseAction(
lambda toks: ptypes.TypeRef(str(toks[0])))
# This is just normal "types", i.e. not channels or messages
typeSpec = Forward()
attributeValue = integer ^ identifier
attribute = Group(
Combine("@" + identifier) +
Optional(lparen + delimitedList(attributeValue) + rparen))
attributes = Group(ZeroOrMore(attribute))
arraySizeSpecImage = Group(image_size_ + lparen + integer + comma +
identifier + comma + identifier + rparen)
arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma +
identifier + rparen)
arraySizeSpecCString = Group(cstring_ + lparen + rparen)
arraySizeSpec = lbrack + Optional(
identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes
^ arraySizeSpecCString,
default="") + rbrack
variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \
.setParseAction(parseVariableDef)
switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon)) + variableDef) \
.setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1]))
switchBody = Group(switch_ + lparen + delimitedList(identifier,delim='.', combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \
.setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4]))
messageBody = structBody = Group(lbrace +
ZeroOrMore(variableDef | switchBody) +
rbrace)
structSpec = Group(struct_ + identifier + structBody +
attributes).setParseAction(
lambda toks: ptypes.StructType(
toks[0][1], toks[0][2], toks[0][3]))
# have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type"
typeSpec << (structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_
^ uint32_ ^ int64_ ^ uint64_ ^ typename).setName("type")
flagsBody = enumBody = Group(
lbrace +
delimitedList(Group(enumname + Optional(equals + integer))) +
Optional(comma) + rbrace)
messageSpec = Group(message_ + messageBody + attributes
).setParseAction(lambda toks: ptypes.MessageType(
None, toks[0][1], toks[0][2])) | typename
channelParent = Optional(colon + typename, default=None)
channelMessage = Group(messageSpec + identifier + Optional(equals + integer, default=None) + semi) \
.setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2]))
channelBody = channelParent + Group(lbrace + ZeroOrMore(
server_ + colon | client_ + colon | channelMessage) + rbrace)
enum_ = (enum32_ | enum16_ | enum8_)
flags_ = (flags32_ | flags16_ | flags8_)
enumDef = Group(enum_ + identifier + enumBody + attributes -
semi).setParseAction(lambda toks: ptypes.EnumType(
toks[0][0], toks[0][1], toks[0][2], toks[0][3]))
flagsDef = Group(flags_ + identifier + flagsBody + attributes -
semi).setParseAction(lambda toks: ptypes.FlagsType(
toks[0][0], toks[0][1], toks[0][2], toks[0][3]))
messageDef = Group(message_ + identifier + messageBody + attributes -
semi).setParseAction(
lambda toks: ptypes.MessageType(
toks[0][1], toks[0][2], toks[0][3]))
channelDef = Group(channel_ + identifier + channelBody -
semi).setParseAction(
lambda toks: ptypes.ChannelType(
toks[0][1], toks[0][2], toks[0][3]))
structDef = Group(struct_ + identifier + structBody + attributes -
semi).setParseAction(lambda toks: ptypes.StructType(
toks[0][1], toks[0][2], toks[0][3]))
typedefDef = Group(typedef_ + identifier + typeSpec + attributes -
semi).setParseAction(lambda toks: ptypes.TypeAlias(
toks[0][1], toks[0][2], toks[0][3]))
definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef
protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi) \
.setParseAction(lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2]))
protocolDef = Group(protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi) \
.setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2]))
bnf = ZeroOrMore(definitions) + protocolDef + StringEnd()
singleLineComment = "//" + restOfLine
bnf.ignore(singleLineComment)
bnf.ignore(cStyleComment)
return bnf
class SFZParser(object):
def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section
^ opcode
^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def handle_include(self, s, loc, toks):
path = os.path.join(self.base_path, normalize_path(toks[0].value))
try:
with open(path) as fp:
f = fp.read()
except IOError as exc:
raise IncludeException(
s, loc=toks[0].locn_start, msg=str(exc))
subparser = SFZParser(path, f, self.state)
subparser.sfz_file.parseString(f)
def handle_section(self, s, loc, toks):
name = toks[0]
if name == 'region':
section = Region(self.state.instr, name, group=self.state.current_group, control=self.state.current_control)
self.state.instr.regions.append(section)
elif name == 'group':
section = Section(self.state.instr, name)
self.state.current_group = section
elif name == 'control':
section = Section(self.state.instr, name)
self.state.current_control = section
else:
raise InvalidSectionException(
s, loc, "Invalid section name '%s'" % name)
self.state.current_section = section
def handle_opcode(self, s, loc, toks):
loc = toks[0].locn_start
name = toks[0].value
try:
opdef = opmap[name]
except KeyError:
raise UnknownOpCodeException(
s, loc=loc, msg="Unknown opcode '%s'" % key)
try:
value = opdef.parse_value(toks[1])
except ValueError as exc:
raise InvalidValueException(
s, loc=loc,
msg="Invalid value for opcode '%s': %s" % (key, str(exc)))
self.state.current_section._opcodes[name] = value
self.state.current_section._opcode_locs[name] = (s, loc)
def parse(self):
self.state = ParserState()
self.state.instr = Instrument(os.path.abspath(self.path))
self.sfz_file.parseString(self.text)
for region in self.state.instr.regions:
if not os.path.isfile(region.sample):
s, loc = region.get_opcode_loc('sample')
raise SampleMissingException(
s, loc, "Missing sample '%s'" % region.sample)
return self.state.instr
optionDirective = OPTION_ - IDENT("optionName") + EQ + quotedString(
"optionValue") + SEMI
#annotateDef = AT + (IDENT + EQ + (quotedString | STRQ3)).setParseAction(addToAnnotationDict)
annotateDef = AT + (IDENT + EQ + ANNOTSTR).setParseAction(addToAnnotationDict)
#topLevelStatement = Group(structDefn | structExtension | enumDefn | serviceDefn | importDirective | optionDirective)
##topLevelStatement = Group(structDefn | structExtension | enumDefn | importDirective | optionDirective)
topLevelStatement = Group(annotateDef | structDefn | enumDefn | importDirective
| optionDirective)
##parser = Optional(packageDirective) + ZeroOrMore(topLevelStatement)
#parser = Group(CMNT) + ZeroOrMore(topLevelStatement)
parser = ZeroOrMore(topLevelStatement)
parser.ignore(comment)
test1 = """
@name = "Test BBX"
@version = "0.1-4"
@doc_title = "BBX document"
@doc_header = "BBX document heading generation Testing"
@doc_intro = '''This is the definiton of the message protocol
used for bla-bla-bla'''
@c_includes = '''
#include <stdio.h>
#include "comms.h"
'''
@c_code = '''
ddlConstraint = Or([
ddlWord("CONSTRAINT"),
ddlWord("PRIMARY"),
ddlWord("FOREIGN"),
ddlWord("KEY"),
ddlWord("INDEX"),
ddlWord("UNIQUE"),
])
ddlColumn = Group(Optional(ddlConstraint).setResultsName("isConstraint") + OneOrMore(MatchFirst([ddlNotNull, ddlAutoValue, ddlDefaultValue, ddlTerm, ddlNum, ddlColumnComment, ddlString, ddlArguments])))
createTable = Group(ddlWord("CREATE") + ddlWord("TABLE") + ddlTerm.setResultsName("tableName") + "(" + Group(delimitedList(ddlColumn)).setResultsName("columns") + ")").setResultsName("create")
ddl = ZeroOrMore(SkipTo(createTable, True))
ddlComment = oneOf(["--", "#"]) + restOfLine
ddl.ignore(ddlComment)
# MAP SQL TYPES
types = {
'tinyint': 'tinyint',
'smallint': 'smallint',
'integer': 'integer',
'int': 'integer',
'bigint': 'bigint',
'char': 'char_',
'varchar': 'varchar',
'text': 'text',
'tinyblob': 'blob',
'blob': 'blob',
'mediumblob': 'blob',
'longblob': 'blob',
def make_grammar_2():
"""
Construct the BBDB grammar. See grammar.ebnf for the specification.
"""
# Helper functions for the brace types.
LP, RP, LB, RB = map(Suppress, "()[]")
Paren = lambda arg: LP + Group(arg) + RP
Bracket = lambda arg: LB + Group(arg) + RB
# Helper functions for constructing return types.
def make_list(t):
return t.asList()
def make_dict(t):
return {k: v for k, v in t[0] or []}
def make_address_entry(t):
return t[0].tag, {
"location": list(t[0].location or []),
"city": t[0].city or "",
"state": t[0].state or "",
"zipcode": t[0].zipcode or "",
"country": t[0].country or ""
}
def make_record(t):
return {
"firstname": t[0].firstname,
"lastname": t[0].lastname,
"aka": t[0].aka or [],
"company": t[0].company or "",
"phone": t[0].phone or {},
"address": t[0].address or {},
"net": t[0].net or [],
"fields": t[0].fields or {}
}
def make_string(t):
return t[0][1:-1].replace(r'\"', '"')
# Define the low-level entities.
string = QuotedString(quoteChar='"', escChar='\\', unquoteResults=False)
string.setParseAction(make_string)
nil = Keyword("nil")
nil.setParseAction(lambda t: [None])
atom = Word(alphanums + '-')
dot = Suppress(Keyword("."))
integer = Word(nums)
integer.setParseAction(lambda t: int(t[0]))
# Phone.
phone_usa = Group(OneOrMore(integer))
phone_nonusa = string
phone_entry = Bracket(string("tag") + Or([phone_usa, phone_nonusa]))
phone = Or([Paren(OneOrMore(phone_entry)), nil])("phone")
phone.setParseAction(make_dict)
# Address.
location = Paren(OneOrMore(string))("location")
location.setParseAction(make_list)
address_entry = Bracket(
string("tag") + location + string("city") + string("state") +
string("zipcode") + string("country"))
address_entry.setParseAction(make_address_entry)
address = Or([Paren(OneOrMore(address_entry)), nil])("address")
address.setParseAction(make_dict)
# Field.
field = Paren(atom + dot + string)
fields = Or([Paren(OneOrMore(field)), nil])("fields")
fields.setParseAction(make_dict)
# Other parts of an entry.
name = string("firstname") + Or([string("lastname"), nil])
company = Or([string, nil])("company")
aka = Or([Paren(OneOrMore(string)), nil])("aka")
aka.setParseAction(make_list)
net = Or([Paren(OneOrMore(string)), nil])("net")
net.setParseAction(make_list)
cache = nil("cache")
# A single record.
record = Bracket(name + aka + company + phone + address + net + fields +
cache)
record.setParseAction(make_record)
# All the records.
bbdb = ZeroOrMore(record)
bbdb.setParseAction(make_list)
# Define comment syntax.
comment = Regex(r";.*")
bbdb.ignore(comment)
return bbdb
def read_chain(s, struct_name, enum_def_parser, struct_def_parser,
header_assignment_parser, struct_parsers):
"""Return a list of dictionaries with the contents of structures in a Yanny par file.
:Parameters:
- `s`: the string to parse
- `struct_name`: the name of the struct to extract
- `enum_def_parser`: a parser that parses one enum definition
- `struct_def_parser`: a parser that parses on struct definition
- `header_assignment_parser`: a parser that parses header assignments
- `struct_parsers`: a dictionary of structures that parse structure data
@return: a list of dictionaries with the contents of a chain
>>> test_string = \"""# A sample file
... a foo
... bar baz
...
... typedef enum {
... OAK,
... MAPLE
... } TREETYPE;
...
... typedef struct {
... float x;
... int y
... } GOO
...
... typedef struct {
... int i;
... TREETYPE s
... } TREE;
...
... GOO 3.4 6
... TREE 42 MAPLE
... TREE 44 OAK
... TREE 3 MAPLE
... GOO 4.22 103
... \"""
>>>
>>> enum_def_parser = make_one_enum_def_parser()
>>> type_parsers = parse_enum_defs(test_string, base_type_parsers)
>>> struct_def_parser = make_one_struct_def_parser(type_parsers)
>>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers)
>>> header_assignment_parser = make_one_header_assignment_parser(struct_parsers)
>>> trees = read_chain(test_string, 'TREE',
... enum_def_parser, struct_def_parser, header_assignment_parser, struct_parsers)
>>> print len(trees)
3
>>> print trees[1]['i']
44
>>> print trees[2]['s']
MAPLE
"""
struct_name = struct_name.upper()
other_list_parsers = [
struct_parsers[sn] for sn in struct_parsers.keys()
if not sn == struct_name
]
not_this_list_parser = Or(
other_list_parsers +
[enum_def_parser, header_assignment_parser, struct_def_parser]
).suppress()
this_list_parser = ZeroOrMore(not_this_list_parser
| struct_parsers[struct_name]) + stringEnd
this_list_parser.ignore(hash_comment)
this_list_parser.ignore(cStyleComment)
raw_results = this_list_parser.parseString(s)
results = []
for row_result in raw_results:
dict_result = {}
for field, value in row_result.items():
dict_result[field] = value
results.append(dict_result)
return results
class PulseProgram:
""" Encapsulates a PulseProgrammer Program
loadSource( filename ) loads the contents of the file
The code is compiled in the following steps
parse() generates self.code
toBytecode() generates self.bytecode
toBinary() generates self.binarycode
the procedure updateVariables( dictionary ) updates variable values in the bytecode
"""
def __init__(self):
self.variabledict = collections.OrderedDict() # keeps information on all variables to easily change them later
self.labeldict = dict() # keep information on all labels
self.source = collections.OrderedDict() # dictionary of source code files (stored as strings)
self.code = [] # this is a list of lines
self.bytecode = [] # list of op, argument tuples
self.binarycode = bytearray() # binarycode to be uploaded
self._exitcodes = dict() # generate a reverse dictionary of variables of type exitcode
self.constDict = dict()
class Board:
channelLimit = 1
halfClockLimit = 500000000
self.adIndexList = [(x, 0) for x in range(6) ]
self.adBoards = [ Board() ]*6
self.timestep = Q(20.0, 'ns')
self.initBNF()
def initBNF(self):
constdecl = (CONST + NAME + VALUE).setParseAction(self.const_action)
vardecl = (VAR + NAME + VALUE + Optional( COMMA + Regex("[^#\n]*")) ).setParseAction(self.var_action)
insertdecl = (INSERT + dblQuotedString + LineEnd().suppress()).setParseAction(self.insert_action)
LABEL = IDENTIFIER + COLON
COMMANDEXP = (IDENTIFIER.setWhitespaceChars(" \t") + Regex("[^#\n]*").setWhitespaceChars(" \t") + LineEnd().suppress() )
COMMAND = COMMANDEXP.setParseAction(self.command_action)
LABELEDCOMMAND = (LABEL + COMMANDEXP ).setParseAction(self.label_command_action)
decl = constdecl | vardecl | insertdecl | LABELEDCOMMAND | COMMAND
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def const_action( self, text, loc, arg ):
""" add the const to the self.constDict dictionary
"""
logger = logging.getLogger(__name__)
logger.debug("{0}:{1} const {2}".format(self.currentFile, lineno(loc, text), arg))
label, value = arg
if label in self.constDict:
logger.error( "Error parsing const in file '{0}': attempted to redefine'{1}' to '{2}' from '{3}'".format(self.currentFile, label, value, self.constDict[label]) )
raise ppexception("Redefining variable", self.currentFile, lineno, label)
else:
self.constDict[label] = int(value)
def var_action( self, text, loc, arg):
print("var_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "")
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug( "{0}:{1} Variable {2}".format( self.currentFile, lineno(loc, text), arg ) )
var = Variable()
label, data = arg[:2]
fields = arg[2].split(",") if len(arg)>2 else [None]*3
fields += [None]*(3-len(fields))
var.type, unit, var.encoding = [ x if x is None or '' else x.strip() for x in fields ]
var.name = label
var.origin = self.currentFile
var.enabled = True
if var.encoding not in encodings:
raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, self.currentFile, lineno(loc, text)), self.currentFile, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(self.currentFile, data) )
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter( var.value, var.encoding )
else:
var.value = Q(float(data))
data = int(round(float(data)))
if label in self.defines:
logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(self.currentFile, label, data, self.defines[label]) )
raise ppexception("variable redifinition", self.currentFile, lineno, label)
else:
self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({ label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
def command_action( self, text, loc, arg):
print("command_action", self.currentFile, lineno(loc, text), arg[0:1], arg[1].split(",") if len(arg)>1 else "")
def label_command_action( self, text, loc, arg):
print("label_command_action", self.currentFile, lineno(loc, text), arg[0:2], arg[2].split(",") if len(arg)>2 else "")
def addLabel(self, label, address, sourcename, lineno):
if label is not None:
self.labeldict[label] = address
def insert_action( self, text, loc, arg ):
oldfile = self.currentFile
print("insert_action", lineno(loc, text), arg)
myprogram = self.program.copy()
self.currentFile = arg[0][1:-1]
result = myprogram.parseFile( self.currentFile )
self.currentFile = oldfile
print(result)
return result
def assembleFile(self, filename):
self.currentFile = filename
result = self.program.parseFile( self.currentFile )
return result
def setHardware(self, adIndexList, adBoards, timestep ):
self.adIndexList = adIndexList
self.adBoards = adBoards
self.timestep = timestep
assert self.timestep.has_dimension('s')
def saveSource(self):
for name, text in self.source.items():
with open(os.path.join(self.pp_dir, name), 'w') as f:
f.write(text)
def loadSource(self, pp_file):
""" Load the source pp_file
#include files are loaded recursively
all code lines are added to self.sourcelines
for each source file the contents are added to the dictionary self.source
"""
self.source.clear()
self.pp_dir, self.pp_filename = os.path.split(pp_file)
self.sourcelines = []
self.insertSource(self.pp_filename)
self.compileCode()
def updateVariables(self, variables ):
""" update the variable values in the bytecode
"""
logger = logging.getLogger(__name__)
for name, value in variables.items():
if name in self.variabledict:
var = self.variabledict[name]
address = var.address
var.value = value
logger.debug( "updateVariables {0} at address 0x{2:x} value {1}, 0x{3:x}".format(name, value, address, int(var.data)) )
var.data = self.convertParameter(value, var.encoding )
self.bytecode[address] = (self.bytecode[address][0], var.data )
self.variabledict[name] = var
else:
logger.error( "variable {0} not found in dictionary.".format(name) )
return self.bytecode
def variables(self):
mydict = dict()
for name, var in self.variabledict.items():
mydict.update( {name: var.value })
return mydict
def variable(self, variablename ):
return self.variabledict.get(variablename).value
def variableUpdateCode(self, variablename, value ):
"""returns the code to update the variable directly on the fpga
consists of variablelocation and variablevalue
"""
var = self.variabledict[variablename]
data = self.convertParameter(value, var.encoding )
return bytearray( struct.pack('II', (var.address, data)))
def flattenList(self, l):
return [item for sublist in l for item in sublist]
def variableScanCode(self, variablename, values):
var = self.variabledict[variablename]
# [item for sublist in l for item in sublist] idiom for flattening of list
return self.flattenList( [ (var.address, self.convertParameter(x, var.encoding)) for x in values ] )
def loadFromMemory(self):
"""Similar to loadSource
only this routine loads from self.source
"""
self.sourcelines = []
self._exitcodes = dict()
self.insertSource(self.pp_filename)
self.compileCode()
def toBinary(self):
""" convert bytecode to binary
"""
logger = logging.getLogger(__name__)
self.binarycode = bytearray()
for wordno, (op, arg) in enumerate(self.bytecode):
logger.debug( "{0} {1} {2} {3}".format( hex(wordno), hex(int(op)), hex(int(arg)), hex(int((int(op)<<24) + int(arg))) ) )
self.binarycode += struct.pack('I', int((op<<24) + arg))
return self.binarycode
def currentVariablesText(self):
lines = list()
for name, var in iter(sorted(self.variabledict.items())):
lines.append("{0} {1}".format(name, var.value))
return '\n'.join(lines)
# routines below here should not be needed by the user
insertPattern = re.compile('#insert\s+([\w.-_]+)', re.IGNORECASE)
codelinePattern = re.compile('(#define|\s*[^#\s]+)', re.IGNORECASE)
def insertSource(self, pp_file):
""" read a source file pp_file
calls itself recursively to for #insert
adds the contents of this file to the dictionary self.source
"""
logger = logging.getLogger(__name__)
if pp_file not in self.source:
with open(os.path.join(self.pp_dir, pp_file)) as f:
self.source[pp_file] = ''.join(f.readlines())
sourcecode = self.source[pp_file]
for line, text in enumerate(sourcecode.splitlines()):
m = self.insertPattern.match(text)
if m:
filename = m.group(1)
logger.info( "inserting code from {0}".format(filename) )
self.insertSource(filename)
else:
if self.codelinePattern.match(text):
self.sourcelines.append((text, line+1, pp_file))
labelPattern = re.compile('(\w+):\s+([^#\n\r]*)')
opPattern = re.compile('\s*(\w+)(?:\s+([^#\n\r]*)){0,1}', re.IGNORECASE)
varPattern = re.compile('var\s+(\w+)\s+([^#,\n\r]+)(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:,([^#,\n\r]+)){0,1}(?:#([^\n\r]+)){0,1}') #
def parse(self):
""" parse the code
"""
logger = logging.getLogger(__name__)
self.code = []
self.variabledict = collections.OrderedDict()
self.defines = dict()
addr_offset = 0
for text, lineno, sourcename in self.sourcelines:
m = self.varPattern.match(text)
if m:
self.addVariable(m, lineno, sourcename)
else:
m = self.definePattern.match(text)
if m:
self.addDefine(m, lineno, sourcename)
else:
# extract any JMP label, if present
m = self.labelPattern.match(text)
if m:
label, text = m.groups() #so the operation after ":" still gets parsed CWC 08162012
else:
label = None #The label for non-jump label line is NONE CWC 08172012
# search OPS list for a match to the current line
m = self.opPattern.match(text)
if m:
op, args = m.groups()
op = op.upper()
# split and remove whitespace
arglist = [0] if args is None else [ 0 if x is None else x.strip() for x in args.split(',')]
#substitute the defined variable directly with the corresponding value CWC 08172012
arglist = [ self.defines[x] if x in self.defines else x for x in arglist ]
#check for dds commands so CHAN commands can be inserted
if (op[:3] == 'DDS'):
try:
board = self.adIndexList[int(arglist[0])][0]
except ValueError:
raise ppexception("DDS argument does not resolve to integer", sourcename, lineno, arglist[0])
chan = self.adIndexList[int(arglist[0])][1]
if (self.adBoards[board].channelLimit != 1):
#boards with more than one channel require an extra channel selection command
chanData = self.adBoards[board].addCMD(chan)
chanData += (int(board) << 16)
self.code.append((len(self.code)+addr_offset, 'DDSCHN', chanData, label, sourcename, lineno))
data = arglist if len(arglist)>1 else arglist[0]
self.addLabel( label, len(self.code), sourcename, lineno)
self.code.append((len(self.code)+addr_offset, op, data, label, sourcename, lineno))
else:
logger.error( "Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno) )
raise ppexception("Error processing line {2}: '{0}' in file '{1}' (unknown opcode?)".format(text, sourcename, lineno),
sourcename, lineno, text)
self.appendVariableCode()
return self.code
def appendVariableCode(self):
""" append all variables to the instruction part of the code
"""
for var in list(self.variabledict.values()):
address = len(self.code)
self.code.append((address, 'NOP', var.data if var.enabled else 0, None, var.origin, 0 ))
var.address = address
def addVariable(self, m, lineno, sourcename):
""" add a variable to the self.variablesdict
"""
logger = logging.getLogger(__name__)
logger.debug( "Variable {0} {1} {2}".format( m.groups(), lineno, sourcename ) )
var = Variable()
label, data, var.type, unit, var.encoding, var.comment = [ x if x is None else x.strip() for x in m.groups()]
var.name = label
var.origin = sourcename
var.enabled = True
if var.encoding not in encodings:
raise ppexception("unknown encoding {0} in file '{1}':{2}".format(var.encoding, sourcename, lineno), sourcename, lineno, var.encoding)
try:
data = str(eval(data, globals(), self.defines))
except Exception:
logger.exception( "Evaluation error in file '{0}' on line: '{1}'".format(sourcename, data) )
if unit is not None:
var.value = Q(float(data), unit)
data = self.convertParameter( var.value, var.encoding )
else:
var.value = Q(float(data))
# var.value.output_prec(0) # without dimension the parameter has to be int. Thus, we do not want decimal places :)
data = int(round(float(data)))
if label in self.defines:
logger.error( "Error in file '%s': attempted to reassign '%s' to '%s' (from prev. value of '%s') in a var statement." %(sourcename, label, data, self.defines[label]) )
raise ppexception("variable redifinition", sourcename, lineno, label)
else:
self.defines[label] = label # add the variable to the dictionary of definitions to prevent identifiers and variables from having the same name
# however, we do not want it replaced with a number but keep the name for the last stage of compilation
pass
var.data = data
self.variabledict.update({ label: var})
if var.type == "exitcode":
self._exitcodes[data & 0x0000ffff] = var
# code is (address, operation, data, label or variablename, currentfile)
def toBytecode(self):
""" generate bytecode from code
"""
logger = logging.getLogger(__name__)
logger.debug( "\nCode ---> ByteCode:" )
self.bytecode = []
for line in self.code:
logger.debug( "{0}: {1}".format(hex(line[0]), line[1:] ))
bytedata = 0
if line[1] not in OPS:
raise ppexception("Unknown command {0}".format(line[1]), line[4], line[5], line[1])
byteop = OPS[line[1]]
try:
data = line[2]
#attempt to locate commands with constant data
if (data == ''):
#found empty data
bytedata = 0
elif isinstance(data, (int, int)):
bytedata = data
elif isinstance(data, float):
bytedata = int(data)
elif isinstance(data, str): # now we are dealing with a variable and need its address
bytedata = self.variabledict[line[2]].address if line[2] in self.variabledict else self.labeldict[line[2]]
elif isinstance(data, list): # list is what we have for DDS, will have 8bit channel and 16bit address
channel, data = line[2]
if isinstance(data, str):
data = self.variabledict[data].address
bytedata = ((int(channel) & 0xf) << 16) | (int(data) & 0x0fff)
except KeyError:
logger.error( "Error assembling bytecode from file '{0}': Unknown variable: '{1}'. \n".format(line[4], data) )
raise ppexception("{0}: Unknown variable {1}".format(line[4], data), line[4], line[5], data)
self.bytecode.append((byteop, bytedata))
logger.debug( "---> {0} {1}".format(hex(byteop), hex(bytedata)) )
return self.bytecode
def convertParameter(self, mag, encoding=None ):
""" convert a dimensioned parameter to the binary value
expected by the hardware. The conversion is determined by the variable encoding
"""
if is_Q(mag):
if mag.dimensionality == Dimensions.time:
result = int(round(mag / self.timestep))
else:
step, unit, _, mask = encodings[encoding]
result = int(round(mag.m_as(unit) / step)) & mask
else:
if encoding:
step, unit, _, mask = encodings[encoding]
result = int(round(mag/step)) & mask
else:
result = mag
return result
def compileCode(self):
self.parse()
self.toBytecode()
def exitcode(self, code):
if code in self._exitcodes:
var = self._exitcodes[code]
if var.comment:
return var.comment
else:
return var.name
else:
return "Exitcode {0} Not found".format(code)
include = INCLUDE + quotedString
# -- the record itself
record = brace_group(RECORDTYPE, field | include | c_code, types=(RTYP, ))
# -- all top-level .dbd entries
device_line = paren_group(DEVICE, RTYP, link_type, dset_name, quotedString)
driver_line = paren_group(DRIVER, c_identifier)
registrar_line = paren_group(REGISTRAR, c_identifier)
function_line = paren_group(FUNCTION, c_identifier)
variable_line = paren_group(VARIABLE, c_identifier,
Optional(c_identifier, default='int'))
top_level = Group(record | menu | device_line | driver_line | registrar_line
| variable_line | function_line)
dbd = ZeroOrMore(top_level)
dbd.ignore(pythonStyleComment)
# -- include files (supported only for including additional fields)
dbd_include = ZeroOrMore(field | include | c_code)
dbd_include.ignore(pythonStyleComment)
def iter_results(info):
check = [info]
while check:
c = check.pop(0)
if c:
if isinstance(c[0], str):
yield c
else:
for other in c:
def from_gml(text):
"""
Return the list of graphs read from `text`, a string in GML format.
This function assumes that the input follows the GML specification,
provides unique integer ids even for isolated nodes, and
defines one or more graphs.
This function ignores anything other than node ids and edge endpoints.
This means directed graphs are read as undirected graphs,
node labels and edge weights are discarded, etc.
If an edge endpoint (integer) is an unknown node id, the node is created.
"""
# Define the grammar with [pyparsing](http://pyparsing.wikispaces.com).
# Don't use `from pyparsing import *` as it adds many constants
# to the generated documentation.
from pyparsing import (
srange, oneOf, Forward, Optional, Suppress, Word, ZeroOrMore,
dblQuotedString, pythonStyleComment
)
digit = srange("[0-9]")
sign = Optional(oneOf("+ -"))
mantissa = Optional("E" + sign + digit)
# `Word(x)` is a sequence of one or more characters from the set x.
digits = Word(digit)
integer = sign + digits
real = sign + Optional(digits) + "." + Optional(digits) + mantissa
# For simplicity, use pyparsing's string with double-quotes,
# hoping that it is a generalisation of GML's definition of a string.
string = dblQuotedString
# A GML file is a list of key-value pairs, where a value may be a list.
# To handle this recursive definition, we delay what a pair is.
pair = Forward()
list = ZeroOrMore(pair)
# A file may have comments, which are as in Python. Ignore them.
list.ignore(pythonStyleComment)
# `Word(x, y)` is 1 character from x followed by 0 or more from y.
key = Word(srange("[a-zA-Z]"), srange("[a-zA-Z0-9]"))
# `Suppress(x)` matches x but doesn't put it in the list of parsed tokens.
listValue = Suppress("[") + list + Suppress("]")
value = real | integer | string | listValue
# The mandatory key-value pairs for graphs are as follows.
graph = Suppress("graph") + listValue
node = Suppress("node") + listValue
anEdge = "edge" + listValue # to avoid conflict with edge() function
id = Suppress("id") + integer
source = Suppress("source") + integer
target = Suppress("target") + integer
# First try to parse graph-specific key-value pairs; otherwise ignore pair.
pair <<= graph | node | anEdge | id | source | target | Suppress(key+value)
# The above suppressions lead to the GML string
# `'graph [ node [id 1 label "ego"] edge [source 1 target 1 weight 0.5] ]'`
# being parsed into the list of tokens
# `["1", "edge", "1", "1"]`,
# which is converted by the following functions into a graph.
def to_int(text, position, tokens):
# Convert parsed integer tokens to integers, e.g. `["1"]` to `1`.
return int(tokens[0])
def to_edge(text, position, tokens):
# Assuming the above conversion was done,
# convert `["edge", a, b]` to an edge incident to a and b.
return edge(tokens[1], tokens[2])
def to_graph(text, position, tokens):
# `tokens` is now a list of integers and edges, in any order.
nodes = set()
edges = set()
for token in tokens:
# If the token is an integer, it's a node id.
if isinstance(token, int):
nodes.add(token)
else:
edges.add(token)
return network(edges, nodes)
# Do the conversions as soon as the respective tokens are parsed.
integer.setParseAction(to_int)
anEdge.setParseAction(to_edge)
graph.setParseAction(to_graph)
# Parse the text with the main grammar rule.
# Return the result as a list, not as a pyparsing object.
return list.parseString(text).asList()
Group(OneOrMore(param_record)).setResultsName('records') + END
param_stmt.setParseAction(ParamStmt)
param_tabbing_stmt = Keyword("param") + param_default + ':' + Optional(symbol + ': ') + \
OneOrMore(data).setResultsName('params') \
+ ':=' + ZeroOrMore(single).setResultsName('data') + END
param_tabbing_stmt.setParseAction(ParamTabbingStmt)
param_def_stmt = Keyword("param") + symbol + Optional(subscript_domain) + \
param_default + END
param_def_stmt.setParseAction(ParamDefStmt)
stmts = set_stmt | set_def_stmt | param_stmt | param_def_stmt | \
param_tabbing_stmt
grammar = ZeroOrMore(stmts) + StringEnd()
grammar.ignore("#" + SkipTo(lineEnd))
grammar.ignore("end;" + SkipTo(lineEnd))
class Amply(object):
"""
Data parsing interface
"""
def __init__(self, s=""):
"""
Create an Amply parser instance
@param s (default ""): initial string to parse
"""
self.symbols = {}
release_site_definition = Group(identifier.setResultsName('name') + release_site_ + lbrace + dictOf(key,value).setResultsName('entries') + rbrace)
object_definition = Group(identifier.setResultsName('compartmentName') + Suppress(object_) + (bracketidentifier | identifier) + (nestedExpr('{', '}',content=statement)).setResultsName('compartmentOptions'))
hashed_initialization_section = Group(hashsymbol + Suppress(instantiate_) + identifier.setResultsName('name') +
identifier.setResultsName('type') + lbrace + Group(ZeroOrMore(release_site_definition | object_definition)).setResultsName('entries') + rbrace )
other_sections = section_enclosure_
#statement = Group(identifier + equal + (quotedString | OneOrMore(mathElements))) + Suppress(LineEnd() | StringEnd())
grammar = ZeroOrMore(Suppress(other_sections) | Suppress(statement) | hashed_system_constants.setResultsName('systemConstants')
| hashed_molecule_section.setResultsName('molecules') | hashed_reaction_section.setResultsName('reactions')
| hashed_observable_section.setResultsName('observables')
| hashed_initialization_section.setResultsName('initialization')
| hashed_function_section.setResultsName('math_functions')
#| Suppress(hashed_section)
)
nonhashedgrammar = ZeroOrMore(Suppress(statement) | Suppress(hashed_section) | Dict(other_sections))
statementGrammar = ZeroOrMore(statement | Suppress(other_sections) | Suppress(hashed_section))
singleLineComment = "//" + restOfLine
grammar.ignore(singleLineComment)
grammar.ignore(cppStyleComment)
nonhashedgrammar.ignore(singleLineComment)
nonhashedgrammar.ignore(cppStyleComment)
statementGrammar.ignore(singleLineComment)
statementGrammar.ignore(cppStyleComment)
def get_parser(self):
declaration = Forward()
keyword = (
Keyword("enum") |
Keyword("case") |
Keyword("struct") |
Keyword("default") |
Keyword("switch") |
Keyword("union") |
Keyword("const") |
Keyword("unsigned") |
Keyword("int") |
Keyword("hyper") |
Keyword("float") |
Keyword("double") |
Keyword("bool") |
Keyword("typedef") |
Keyword("opaque") |
Keyword("string") |
Keyword("void") |
Keyword("program") |
Keyword("version")
)
identifier = NotAny(keyword) + Word(alphas + alphas.upper(), alphanums + alphanums.upper() + "_", asKeyword=True)
constant = Combine(Optional("-") + Word(nums))
constant.setParseAction(lambda s,l,t: [int(t[0])])
value = constant | identifier
enum_body = Literal("{").suppress() + identifier + Literal("=").suppress() + value + ZeroOrMore(Literal(",").suppress() + identifier + Literal("=").suppress() + value) + Literal("}").suppress()
enum_type_spec = Literal("enum").suppress() + enum_body
enum_body.setParseAction(self.parse_enum)
struct_body = Literal("{").suppress() + OneOrMore(declaration + Literal(";").suppress()) + Literal("}").suppress()
struct_type_spec = Literal("struct").suppress() + struct_body
struct_body.setParseAction(self.parse_struct)
case_stmt = Literal("case").suppress() + value + Literal(":").suppress() + declaration + Literal(";").suppress()
default_stmt = Literal("default") + Literal(":").suppress() + declaration + Literal(";").suppress()
union_body = Literal("switch").suppress() + Literal("(").suppress() + declaration + Literal(")").suppress() + Literal("{").suppress() + Group(OneOrMore(Group(case_stmt)) + Optional(Group(default_stmt))) + Literal("}").suppress()
union_type_spec = Literal("union").suppress() + union_body
union_body.setParseAction(self.parse_union)
constant_def = Literal("const").suppress() + identifier + Literal("=").suppress() + constant + Literal(";").suppress()
constant_def.setParseAction(self.parse_const)
type_spec = (
(Optional(Literal("unsigned")) + Literal("int")).setParseAction(self.parse_builtin) |
(Optional(Literal("unsigned")) + Literal("hyper")).setParseAction(self.parse_builtin) |
Literal("float").setParseAction(self.parse_builtin) |
Literal("double").setParseAction(self.parse_builtin) |
Literal("bool").setParseAction(self.parse_builtin) |
enum_type_spec |
struct_type_spec |
union_type_spec |
identifier
)
proc_return = Literal("void") | type_spec
procedure_def = proc_return + identifier + Literal("(").suppress() + (Literal("void") | type_spec) + ZeroOrMore(Literal(",").suppress() + type_spec) + Literal(")").suppress() + Literal("=").suppress() + constant + Literal(";").suppress()
procedure_def.setParseAction(self.parse_procedure_def)
version_def = Literal("version").suppress() + identifier + Literal("{").suppress() + OneOrMore(procedure_def) + Literal("}").suppress() + Literal("=").suppress() + constant + Literal(";").suppress()
version_def.setParseAction(self.parse_version_def)
program_body = Literal("{").suppress() + Group(OneOrMore(version_def)) + Literal("}").suppress()
type_def = (
(Literal("typedef") + declaration + Literal(";")) |
(Literal("enum") + identifier + enum_body + Literal(";")) |
(Literal("struct") + identifier + struct_body + Literal(";")) |
(Literal("union") + identifier + union_body + Literal(";")) |
(Literal("program") + identifier + program_body + Literal("=").suppress() + constant + Literal(";"))
)
type_def.setParseAction(self.parse_type_def)
declaration << (
(type_spec + identifier + Literal("[") + value + Literal("]")) |
(type_spec + identifier + Literal("<") + value + Literal(">")) |
(type_spec + identifier) |
(Literal("opaque") + identifier + Literal("[") + value + Literal("]")) |
(Literal("opaque") + identifier + Literal("<") + value + Literal(">")) |
(Literal("string") + identifier + Literal("<") + value + Literal(">")) |
(type_spec + Literal("*") + identifier) |
Literal("void")
)
declaration.setParseAction(self.parse_decl)
definition = type_def | constant_def
specification = ZeroOrMore(definition)
comment = (Literal("#") + restOfLine).suppress()
specification.ignore(comment)
return specification
from pyparsing import alphas, alphanums, hexnums, cppStyleComment, delimitedList, dblQuotedString, \
Suppress, Keyword, Word, Regex, Group, Optional, ZeroOrMore
semi = Suppress(';')
token_lbra = Suppress('{')
token_rbra = Suppress('}')
token_type = Keyword('byte') | Keyword('word')
token_id = Word(alphas + '_', alphanums + '_')
token_integer = Regex(r'[+-]?\d+')
token_hex = Suppress('0x') + Word(hexnums)
const_array = Group(Suppress('[') + delimitedList(token_hex) + Suppress(']'))
const_value = token_integer | const_array
var = Group(token_type + delimitedList(Group(token_id + Optional('=' + const_value)))) + semi
call_func = Group(token_id + Suppress('()'))('call') + semi
set_register = Group(token_hex + '=' + const_value) + semi
body = ZeroOrMore(call_func | set_register | Group(dblQuotedString))
func = Group(Suppress('function') + token_id + Suppress('()') + token_lbra + Group(body) + token_rbra)('func')
const = Group(Suppress('const') + token_type +
Group(delimitedList(
Group(token_id + '=' + const_value)
)) + semi)('const')
nes = ZeroOrMore(const | var | func)
nes.ignore(cppStyleComment)
key = identifier + Suppress('=')
value = restOfLine
release_site_definition = Group(identifier.setResultsName('name') + release_site_ + lbrace + dictOf(key, value).setResultsName('entries') + rbrace)
object_definition = Group(identifier.setResultsName('compartmentName') + Suppress(object_) + (bracketidentifier | identifier) + (nestedExpr('{', '}', content=statement)).setResultsName('compartmentOptions'))
hashed_initialization_section = Group(hashsymbol + Suppress(instantiate_) + identifier.setResultsName('name') +
identifier.setResultsName('type') + lbrace + Group(ZeroOrMore(release_site_definition | object_definition)).setResultsName('entries') + rbrace)
other_sections = section_enclosure_
# statement = Group(identifier + equal + (quotedString | OneOrMore(mathElements))) + Suppress(LineEnd() | StringEnd())
grammar = ZeroOrMore(Suppress(other_sections) | Suppress(statement) | hashed_system_constants.setResultsName('systemConstants') |
hashed_molecule_section.setResultsName('molecules') | hashed_reaction_section.setResultsName('reactions') |
hashed_observable_section.setResultsName('observables') |
hashed_initialization_section.setResultsName('initialization') |
hashed_function_section.setResultsName('math_functions')
# | Suppress(hashed_section)
)
nonhashedgrammar = ZeroOrMore(Suppress(statement) | Suppress(hashed_section) | Dict(other_sections))
statementGrammar = ZeroOrMore(statement | Suppress(other_sections) | Suppress(hashed_section))
singleLineComment = "//" + restOfLine
grammar.ignore(singleLineComment)
grammar.ignore(cppStyleComment)
nonhashedgrammar.ignore(singleLineComment)
nonhashedgrammar.ignore(cppStyleComment)
statementGrammar.ignore(singleLineComment)
statementGrammar.ignore(cppStyleComment)
selectStatement << ( selectToken +
columnNameList.setResultsName("columns") +
fromToken +
identifier.setResultsName("tablename") +
Optional(whereClause) +
Optional(orderByClause) +
Optional(limitClause) )
BQLStatement = (selectStatement | createBtableStatement) + Optional(';')
BQL = ZeroOrMore(BQLStatement)
## allows comments
dashComment = "--" + restOfLine
BQL.ignore(dashComment)
def test( str ):
print str,"->"
try:
tokens = BQL.parseString( str )
print "tokens = ", tokens
print "tokens.tablename =", tokens.tablename
print "tokens.filename =", tokens.filename
#print "tokens.where =", tokens.where
except ParseException, err:
print " "*err.loc + "^\n" + err.msg
print err
print
def __init__(self, processor, baseiri, strict=False):
"""
See class docstring.
"""
# pylint: disable=R0914,R0915
self.reset(processor, baseiri, strict)
PrefixedName = PNAME_LN | PNAME_NS
Iri = IRIREF | PrefixedName
BNode = BLANK_NODE_LABEL | ANON
RDFLiteral = STRING + Optional(LANGTAG("langtag") | Group(Suppress("^^") + Iri)("datatype"))
Object = Forward()
Collection = Suppress("(") + ZeroOrMore(Object) + Suppress(")")
PredicateObjectList = Forward()
BlankNodePropertyList = Suppress("[") + PredicateObjectList + Suppress("]")
TtlLiteral = RDFLiteral | NUMERIC_LITERAL | BOOLEAN_LITERAL
Subject = Iri | BNode | Collection | VARIABLE # added for LD Patch
Predicate = Iri
Object << ( # pylint: disable=W0104
Iri | BNode | Collection | BlankNodePropertyList | TtlLiteral | VARIABLE
) # added for LD Patch
Verb = Predicate | Keyword("a")
ObjectList = Group(Object + ZeroOrMore(COMMA + Object))
PredicateObjectList << ( # pylint: disable=W0106
Verb + ObjectList + ZeroOrMore(SEMICOLON + Optional(Verb + ObjectList))
)
Triples = (Subject + PredicateObjectList) | (BlankNodePropertyList + Optional(PredicateObjectList))
Value = Iri | TtlLiteral | VARIABLE
InvPredicate = Suppress("^") + Predicate
Step = Suppress("/") + (Predicate | InvPredicate | INDEX)
Filter = Forward()
Constraint = Filter | UNICITY_CONSTRAINT
Path = Group(OneOrMore(Step | Constraint))
Filter << (
Suppress("[") # pylint: disable=W0106
+ Group(ZeroOrMore(Step | Constraint))("path") # = Path (*)
+ Optional(Suppress("=") + Object)("value")
+ Suppress("]")
)
# (*) we can not reuse the Path rule defined above,
# because we want to set a name for that component
Turtle = Triples + ZeroOrMore(PERIOD + Triples) + Optional(PERIOD)
Graph = Suppress("{") + Optional(Turtle) + Suppress("}")
Prefix = Literal("@prefix") + PNAME_NS + IRIREF + PERIOD
if not strict:
SparqlPrefix = CaselessKeyword("prefix") + PNAME_NS + IRIREF
Prefix = Prefix | SparqlPrefix
Bind = BIND_CMD + VARIABLE + Value + Optional(Path) + PERIOD
Add = ADD_CMD + Graph + PERIOD
AddNew = ADDNEW_CMD + Graph + PERIOD
Delete = DELETE_CMD + Graph + PERIOD
DeleteExisting = DELETEEXISTING_CMD + Graph + PERIOD
Cut = CUT_CMD + VARIABLE + PERIOD
UpdateList = UPDATELIST_CMD + Subject + Predicate + SLICE + Collection + PERIOD
Statement = Prefix | Bind | Add | AddNew | Delete | DeleteExisting | Cut | UpdateList
Patch = ZeroOrMore(Statement)
if not strict:
Patch.ignore("#" + restOfLine) # Comment
Patch.parseWithTabs()
self.grammar = Patch
IRIREF.setParseAction(self._parse_iri)
PrefixedName.setParseAction(self._parse_pname)
RDFLiteral.setParseAction(self._parse_turtleliteral)
Collection.setParseAction(self._parse_collection)
BlankNodePropertyList.setParseAction(self._parse_bnpl)
Verb.setParseAction(self._parse_verb)
ObjectList.setParseAction(self._parse_as_list)
Triples.setParseAction(self._parse_tss)
InvPredicate.setParseAction(self._parse_invpredicate)
Filter.setParseAction(self._parse_filter)
Path.setParseAction(self._parse_as_list)
Prefix.setParseAction(self._do_prefix)
Bind.setParseAction(self._do_bind)
Add.setParseAction(self._do_add)
AddNew.setParseAction(self._do_add_new)
Delete.setParseAction(self._do_delete)
DeleteExisting.setParseAction(self._do_delete_existing)
Cut.setParseAction(self._do_cut)
UpdateList.setParseAction(self._do_updatelist)
def parse_struct_defs(s, type_parsers=None):
"""Parse all the structure definitions in a string
:Parameters:
- `s`: the string to parse
- `type_parsers`: a list of type parsers that can be used
@return: a tuple with two dictionaries, one that describes the structs, the other stores the parsers
>>> test_string = \"""# A sample file
... a foo
... bar baz
...
... typedef struct { # a comment
... float x; /* another comment */
... int y
... } GOO
...
... typedef enum {
... START,
... END
... } RUNMARK;
...
... typedef struct {
... int i;
... RUNMARK m
... } MOO;
...
... GOO 3.4 6
... \"""
>>>
>>> type_parsers = parse_enum_defs(test_string, base_type_parsers)
>>> structs, struct_parsers = parse_struct_defs(test_string, type_parsers)
>>>
>>> print structs.keys()
['GOO', 'MOO']
>>>
>>> print struct_parsers.keys()
['GOO', 'MOO']
>>>
>>> for f in structs['GOO']:
... print "Field name: %-15s type name: %s" % (f['field_name'], f['type_name'])
Field name: x type name: float
Field name: y type name: int
>>>
>>> for f in structs['MOO']:
... print "Field name: %-15s type name: %s" % (f['field_name'], f['type_name'])
Field name: i type name: int
Field name: m type name: RUNMARK
>>>
>>> test_goo = "GOO 3.14 42"
>>> test_parsed_goo = struct_parsers['GOO'].parseString(test_goo)
>>> print "x: %f y: %d" % (test_parsed_goo[0]['x'], test_parsed_goo[0]['y'])
x: 3.140000 y: 42
>>>
>>> test_moo = "MOO 44 END"
>>> test_parsed_moo = struct_parsers['MOO'].parseString(test_moo)
>>> print "i: %d m: %s" % (test_parsed_moo[0]['i'], test_parsed_moo[0]['m'])
i: 44 m: END
"""
if type_parsers is None:
type_parsers = base_type_parsers
one_struct_def_parser = make_one_struct_def_parser(type_parsers)
not_struct = ((value_name | enum_declaration_start | possible_type_name
| right_brace) + restOfLine).suppress()
struct_def_parser = ZeroOrMore(Group(one_struct_def_parser)
| not_struct) + stringEnd
struct_def_parser.ignore(hash_comment)
struct_def_parser.ignore(cStyleComment)
struct = {}
struct_parser = {}
for this_struct in struct_def_parser.parseString(s):
struct_name = this_struct['struct_name'].upper()
struct[struct_name] = \
[{'field_name': f['field_name'], 'type_name': f['type_name']} for f in this_struct['fields']]
struct_parser[struct_name] = \
Group(CaselessKeyword(this_struct['struct_name']) \
+ And( [linecont+type_parsers[f['type_name']](f['field_name'])
for f in this_struct['fields']] ))
return struct, struct_parser
class SFZParser(object):
def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))',
re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal(
'>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(
QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section ^ opcode ^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def handle_include(self, s, loc, toks):
path = os.path.join(self.base_path, normalize_path(toks[0].value))
try:
with open(path) as fp:
f = fp.read()
except IOError as exc:
raise IncludeException(s, loc=toks[0].locn_start, msg=str(exc))
subparser = SFZParser(path, f, self.state)
subparser.sfz_file.parseString(f)
def handle_section(self, s, loc, toks):
name = toks[0]
if name == 'region':
section = Region(self.state.instr,
name,
group=self.state.current_group,
control=self.state.current_control)
self.state.instr.regions.append(section)
elif name == 'group':
section = Section(self.state.instr, name)
self.state.current_group = section
elif name == 'control':
section = Section(self.state.instr, name)
self.state.current_control = section
else:
raise InvalidSectionException(s, loc,
"Invalid section name '%s'" % name)
self.state.current_section = section
def handle_opcode(self, s, loc, toks):
loc = toks[0].locn_start
name = toks[0].value
try:
opdef = opmap[name]
except KeyError:
raise UnknownOpCodeException(s,
loc=loc,
msg="Unknown opcode '%s'" % key)
try:
value = opdef.parse_value(toks[1])
except ValueError as exc:
raise InvalidValueException(
s,
loc=loc,
msg="Invalid value for opcode '%s': %s" % (key, str(exc)))
self.state.current_section._opcodes[name] = value
self.state.current_section._opcode_locs[name] = (s, loc)
def parse(self):
self.state = ParserState()
self.state.instr = Instrument(os.path.abspath(self.path))
self.sfz_file.parseString(self.text)
for region in self.state.instr.regions:
if not os.path.isfile(region.sample):
s, loc = region.get_opcode_loc('sample')
raise SampleMissingException(
s, loc, "Missing sample '%s'" % region.sample)
return self.state.instr
class TOMLParser(object):
def __init__(self):
key_name = Word(re.sub(r"[\[\]=\"]", "", printables))
kgrp_name = Word(re.sub(r"[\[\]\.]", "", printables))
basic_int = Optional("-") + ("0" | Word(nums))
types = dict(
string = QuotedString("\"", escChar="\\"),
integer = Combine(basic_int),
float = Combine(basic_int + "." + Word(nums)),
datetime = Regex(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z"),
boolean = Keyword("true") | Keyword("false"),
array = Forward(),
)
pure_array = Or(delimitedList(type_) for type_ in types.values())
types["array"] << Group(Suppress("[") + Optional(pure_array) +
Suppress("]"))
value = Or(type_ for type_ in types.values())
keyvalue = key_name + Suppress("=") + value + Suppress(LineEnd())
keygroup_namespace = kgrp_name + ZeroOrMore(Suppress(".") + kgrp_name)
keygroup = "[" + keygroup_namespace + "]" + LineEnd()
comments = pythonStyleComment
self._toplevel = ZeroOrMore(keyvalue | keygroup)
self._toplevel.ignore(comments)
for k, v in types.items():
v.setParseAction(getattr(self, "_parse_"+k))
keyvalue.setParseAction(self._parse_keyvalue)
keygroup_namespace.setParseAction(self._parse_keygroup_namespace)
def _parse_string(self, src, loc, toks):
match = re.search(r"(?<!\\)(\\[^0tnr\"\\])", toks[0])
if match:
raise ParseException("Reserved escape sequence \"%s\"" %
match.group(), loc)
return unescape(toks[0])
_parse_integer = lambda self, tok: int(tok[0])
_parse_float = lambda self, tok: float(tok[0])
_parse_boolean = lambda self, tok: bool(tok[0])
ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
def _parse_datetime(self, src, loc, toks):
try:
return datetime.strptime(toks[0], self.ISO8601)
except ValueError:
# this informative error message will never make it out because
# pyparsing catches ParseBaseException and reraises on its own.
# oh well.
raise ParseException("invalid datetime \"%s\"" % toks[0], loc)
_parse_array = lambda self, tok: [tok[0]]
def _parse_keyvalue(self, s, loc, toks):
k, v = toks.asList()
if k in self._cur:
raise ParseException("key %s already exists" % k, loc)
self._cur[k] = v
def _parse_keygroup_namespace(self, s, loc, toks):
cur = self._root
for subname in toks:
subspace = cur.get(subname, {})
if not isinstance(subspace, dict):
raise ParseException("key %s already exists" % subname, loc)
cur = cur.setdefault(subname, subspace)
self._cur = cur
def parse(self, s):
self._root = {}
self._cur = self._root
self._toplevel.parseWithTabs()
self._toplevel.parseString(s, parseAll=True)
return self._root
def SPICE_BNF():
global bnf
if not bnf:
# punctuation
colon = Literal(":").suppress()
lbrace = Literal("{").suppress()
rbrace = Literal("}").suppress()
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
lparen = Literal("(").suppress()
rparen = Literal(")").suppress()
equals = Literal("=").suppress()
comma = Literal(",").suppress()
semi = Literal(";").suppress()
# primitive types
int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8))
uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8))
int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16))
uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16))
int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32))
uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32))
int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64))
uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64))
# keywords
enum32_ = Keyword("enum32").setParseAction(replaceWith(32))
enum16_ = Keyword("enum16").setParseAction(replaceWith(16))
enum8_ = Keyword("enum8").setParseAction(replaceWith(8))
flags32_ = Keyword("flags32").setParseAction(replaceWith(32))
flags16_ = Keyword("flags16").setParseAction(replaceWith(16))
flags8_ = Keyword("flags8").setParseAction(replaceWith(8))
channel_ = Keyword("channel")
server_ = Keyword("server")
client_ = Keyword("client")
protocol_ = Keyword("protocol")
typedef_ = Keyword("typedef")
struct_ = Keyword("struct")
message_ = Keyword("message")
image_size_ = Keyword("image_size")
bytes_ = Keyword("bytes")
cstring_ = Keyword("cstring")
switch_ = Keyword("switch")
default_ = Keyword("default")
case_ = Keyword("case")
identifier = Word(alphas, alphanums + "_")
enumname = Word(alphanums + "_")
integer = (
(Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF")) | Word(nums + "+-", nums))
.setName("int")
.setParseAction(cvtInt)
)
typename = identifier.copy().setParseAction(lambda toks: ptypes.TypeRef(str(toks[0])))
# This is just normal "types", i.e. not channels or messages
typeSpec = Forward()
attributeValue = integer ^ identifier
attribute = Group(Combine("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen))
attributes = Group(ZeroOrMore(attribute))
arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen)
arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen)
arraySizeSpecCString = Group(cstring_ + lparen + rparen)
arraySizeSpec = (
lbrack
+ Optional(
identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^ arraySizeSpecCString, default=""
)
+ rbrack
)
variableDef = Group(
typeSpec
+ Optional("*", default=None)
+ identifier
+ Optional(arraySizeSpec, default=None)
+ attributes
- semi
).setParseAction(parseVariableDef)
switchCase = Group(
Group(
OneOrMore(
default_.setParseAction(replaceWith(None)) + colon
| Group(case_.suppress() + Optional("!", default="") + identifier) + colon
)
)
+ variableDef
).setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1]))
switchBody = Group(
switch_
+ lparen
+ delimitedList(identifier, delim=".", combine=True)
+ rparen
+ lbrace
+ Group(OneOrMore(switchCase))
+ rbrace
+ identifier
+ attributes
- semi
).setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4]))
messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace)
structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction(
lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])
)
# have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type"
typeSpec << (
structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ typename
).setName("type")
flagsBody = enumBody = Group(
lbrace + delimitedList(Group(enumname + Optional(equals + integer))) + Optional(comma) + rbrace
)
messageSpec = (
Group(message_ + messageBody + attributes).setParseAction(
lambda toks: ptypes.MessageType(None, toks[0][1], toks[0][2])
)
| typename
)
channelParent = Optional(colon + typename, default=None)
channelMessage = Group(
messageSpec + identifier + Optional(equals + integer, default=None) + semi
).setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2]))
channelBody = channelParent + Group(
lbrace + ZeroOrMore(server_ + colon | client_ + colon | channelMessage) + rbrace
)
enum_ = enum32_ | enum16_ | enum8_
flags_ = flags32_ | flags16_ | flags8_
enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction(
lambda toks: ptypes.EnumType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])
)
flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction(
lambda toks: ptypes.FlagsType(toks[0][0], toks[0][1], toks[0][2], toks[0][3])
)
messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction(
lambda toks: ptypes.MessageType(toks[0][1], toks[0][2], toks[0][3])
)
channelDef = Group(channel_ + identifier + channelBody + attributes - semi).setParseAction(
lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3], toks[0][4])
)
structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction(
lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3])
)
typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction(
lambda toks: ptypes.TypeAlias(toks[0][1], toks[0][2], toks[0][3])
)
definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef
protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi).setParseAction(
lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2])
)
protocolDef = Group(
protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi
).setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2]))
bnf = ZeroOrMore(definitions) + protocolDef + StringEnd()
singleLineComment = "//" + restOfLine
bnf.ignore(singleLineComment)
bnf.ignore(cStyleComment)
return bnf
pluginName = Word(alphanums + "@_")
pluginDesc = Group(pluginType + pluginName + lbrace + Group(ZeroOrMore(pluginAttr)) + rbrace).setParseAction(
getPluginDesc
)
pluginDesc.ignore("//" + restOfLine)
pluginDesc.ignore(cStyleComment)
# Scene
#
sceneDesc = OneOrMore(pluginDesc)
sceneDesc.ignore("//" + restOfLine)
sceneDesc.ignore(cStyleComment)
nameParser = ZeroOrMore(Group(pluginType + pluginName + lbrace))
nameParser.ignore("//" + restOfLine)
nameParser.ignore(cStyleComment)
def ParseVrscene(filepath):
return sceneDesc.parseString(open(filepath, "r").read())
def GetMaterialsNames(filepath):
materialPluginNames = []
with open(filepath, "r") as f:
for l in f:
result = nameParser.parseString(l)
if result:
res = result[0]
if res[0].startswith("Mtl"):
ddlUnsigned, ddlNotNull, ddlAutoValue, ddlDefaultValue,
ddlFunctionWord("NOW"), ddlTerm, ddlNum, ddlColumnComment, ddlString,
ddlArguments, ddlMathCombined, ddlInlineComment
])))
ddlIfNotExists = Optional(Group(ddlWord("IF") + ddlWord("NOT") + ddlWord(
"EXISTS")).setResultsName("ifNotExists"))
createTable = Group(ddlWord("CREATE") + ddlWord("TABLE") + ddlIfNotExists +
ddlName.setResultsName("tableName") + "(" + Group(
delimitedList(ddlColumn)).setResultsName(
"columns") + ")").setResultsName("create")
#ddlString.setDebug(True) #uncomment to debug pyparsing
ddl = ZeroOrMore(Suppress(SkipTo(createTable, False)) + createTable)
ddlComment = oneOf(["--", "#"]) + restOfLine
ddl.ignore(ddlComment)
# MAP SQL TYPES
types = {
'tinyint': 'tinyint',
'smallint': 'smallint',
'smallserial': 'smallint', # PostgreSQL
'integer': 'integer',
'int': 'integer',
'serial': 'integer', # PostgreSQL
'mediumint': 'integer',
'bigint': 'bigint',
'bigserial': 'bigint', # PostgreSQL
'char': 'char_',
'varchar': 'varchar',
'text': 'text',
def parser_bnf():
"""Grammar for parsing podcast configuration files."""
at = Literal("@").suppress()
caret = Literal("^")
colon = Literal(":").suppress()
left_bracket = Literal("[").suppress()
period = Literal(".").suppress()
right_bracket = Literal("]").suppress()
# zero_index ::= [0-9]+
zero_index = Word(nums).setParseAction(lambda s, l, t: int(t[0]))
# filename ::= [A-Za-z0-9][-A-Za-z0-9._ ]+
filename_first = Word(alphanums, exact=1)
filename_rest = Word(alphanums + "-_/. ")
filename = Combine(filename_first + Optional(filename_rest))
# millisecs ::= "." [0-9]+
millisecs = (Word(nums).setParseAction(
lambda s, l, t: int(t[0][:3].ljust(3, "0"))).setResultsName("ms"))
# hours, minutes, seconds ::= zero_index
hours = zero_index.setResultsName("hh")
minutes = zero_index.setResultsName("mm")
seconds = zero_index.setResultsName("ss")
hours_minutes = hours + colon + minutes + colon | minutes + colon
secs_millisecs = (seconds + Optional(period + millisecs)
| period + millisecs)
# timestamp ::= [[hours ":"] minutes ":"] seconds ["." millisecs]
timestamp = Optional(hours_minutes) + secs_millisecs
# duration_file ::= "@", filename
# We need a separate item for a lonely duration file timestamp so
# that we can attach a parse action just to the lonely case. Using
# duration_file alone means the parse action is attached to all
# instances of duration_file.
duration_file = at + filename.setResultsName("filename")
lonely_duration_file = at + filename.setResultsName("filename")
# timespecs ::= timestamp [duration_file | {timestamp}]
# If duration_file timestamp is lonely, prepend a zero timestamp.
timespecs = Or([
lonely_duration_file.setParseAction(
lambda s, l, t: [timestamp.parseString("00:00:00.000"), t]),
Group(timestamp) + duration_file,
OneOrMore(Group(timestamp.setParseAction(default_timestamp_fields)))
])
# last_frame ::= "-1" | "last"
last_frame = oneOf(["-1", "last"]).setParseAction(replaceWith(-1))
# frame_number ::= ":" (zero_index | last_frame)
frame_number = colon - (zero_index | last_frame).setResultsName("num")
# stream_number ::= ":" zero_index
stream_number = colon - zero_index.setResultsName("num")
# input_file ::= ":" [filename]
input_file = colon - Optional(filename).setResultsName("filename")
# previous_segment ::= ":" "^"
previous_segment = colon - caret.setResultsName("filename")
# frame_input_file ::= input_file | previous_segment
frame_input_file = Or([input_file, previous_segment])
# av_trailer ::= input_file [stream_number]
av_trailer = input_file + Optional(stream_number)
# frame_type ::= "frame" | "f"
frame_type = oneOf(["f", "frame"]).setParseAction(replaceWith("frame"))
# frame_input ::= frame_type [frame_input_file [frame_number]]
frame_input = (frame_type.setResultsName("type") +
Optional(frame_input_file + Optional(frame_number)))
# video_type ::= "video" | "v"
video_type = oneOf(["v", "video"]).setParseAction(replaceWith("video"))
# audio_type ::= "audio" | "a"
audio_type = oneOf(["a", "audio"]).setParseAction(replaceWith("audio"))
# av_input ::= (audio_type | video_type) [av_trailer]
av_input = ((audio_type | video_type).setResultsName("type") +
Optional(av_trailer))
# inputspec ::= "[" (av_input | frame_input) "]"
inputspec = (left_bracket + delimitedList(
av_input | frame_input, delim=":").setParseAction(default_input_fields)
- right_bracket)
# segmentspec ::= inputspec [timespecs]
segmentspec = Group(inputspec +
Group(Optional(timespecs)).setResultsName("times"))
# config ::= {segmentspec}
config = ZeroOrMore(segmentspec)
config.ignore(pythonStyleComment)
return config
