def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
FTR: this is hideous.
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
def _handle_ip(*x):
a,b,c = x[2]
return ' %s = { %s }' % (a,c[0])
def _handle_diraddr(*x):
a,b,c = x[2]
self._set(DIRADDRESSES, ' %s' % '\n '.join(c))
return
def np(words, fn = gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_name = np((NAME,), action=lambda x: self._set_name(x[2]))
gr_address = np((ADDRESS,), action=self._parse_setter(ADDRESS))
gr_fd_conn = np(PList('fd connect timeout'), gr_number, self._parse_setter(FD_CONNECT_TIMEOUT, True))
gr_heart = np(PList('heartbeat interval'), gr_number, self._parse_setter(HEARTBEATINTERVAL, True))
gr_max_con = np(PList('maximum console connections'),
gr_number, self._parse_setter(MAXIMUMCONSOLECONNECTIONS, True))
gr_max_jobs = np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(MAXIMUMCONCURRENTJOBS, True))
gr_pass = np((PASSWORD,), action=self._parse_setter(PASSWORD))
gr_pid = np(PList('pid directory'), action=self._parse_setter(PIDDIRECTORY))
gr_query = np(PList('query file'), action=self._parse_setter(QUERYFILE))
gr_scripts = np(PList('scripts directory'), action=self._parse_setter(SCRIPTS_DIRECTORY))
gr_sd_conn = np(PList('sd connect timeout'), gr_number, self._parse_setter(SD_CONNECT_TIMEOUT, True))
gr_source = np(PList('source address'), action=self._parse_setter(SOURCEADDRESS))
gr_stats = np(PList('statistics retention'), action=self._parse_setter(STATISTICS_RETENTION))
gr_verid = np((VERID,), action=self._parse_setter(VERID))
gr_messages = np((MESSAGES,), action=lambda x:self._parse_setter(MESSAGE_ID, dereference=True))
gr_work_dir = np(PList('working directory'), action=self._parse_setter(WORKINGDIRECTORY))
gr_port = np(PList('dir port'), gr_number, self._parse_setter(PORT, True))
gr_monitor = np((MONITOR,), gr_yn, action=self._parse_setter(MONITOR))
# This is a complicated one
da_addr = np(('Addr','Port'), Word(printables), lambda x,y,z: ' '.join(z))
da_ip = np(('IPv4','IPv6','IP'), nestedExpr('{','}', OneOrMore(da_addr).setParseAction(lambda x,y,z: ' ; '.join(z)))).setParseAction(_handle_ip)
da_addresses = np(PList('dir addresses'), nestedExpr('{','}', OneOrMore(da_ip)), _handle_diraddr)
gr_res = OneOrMore(gr_name | gr_address | gr_fd_conn | gr_heart | gr_max_con | gr_max_jobs | gr_pass | gr_pid | gr_query | gr_scripts | gr_sd_conn | gr_source | gr_stats | gr_verid | gr_messages | gr_work_dir | gr_port | gr_monitor | da_addresses)
result = gr_res.parseString(string, parseAll=True)
return 'Director: ' + self[NAME]
def _grammar(self):
"""
Pyparsing implementation of a where clause grammar based on http://pyparsing.wikispaces.com/file/view/simpleSQL.py
The query language is a series of statements separated by AND or OR operators and parentheses can be used to group/provide
precedence.
A statement is a combination of three strings "<variable> <operator> <value>" or "<value> <operator> <variable>".
A value can be a string, integer or a real(floating) number, a (ISO YYYY-MM-DD) date, or a year based period (nY) where the resulting value
will typically be base_date + relativedelta(years=n).
See self.period_to_date
An operator must be one of "= != < > >= <= !:" and are translated into django __lte or equivalent suffixes.
See self.as_q
Example
something < 10 AND other >= 2015-01-01 AND (foo < 1 OR bar > 1)
"""
quoted_string_excluding_quotes = quotedString.copy().setParseAction(lambda token: StringValue(token[0]))
and_ = Keyword('and', caseless=True)
or_ = Keyword('or', caseless=True)
exponential_marker = CaselessLiteral('E')
binary_op = oneOf('=> =< = < > >= <= : != !:', caseless=True).setResultsName('operator')
arith_sign = Word('+-', exact=1)
integer = Word(nums)
real_num = Combine(Optional(arith_sign) + (integer + '.' + Optional(integer) | ('.' + integer)) + Optional(exponential_marker + Optional(arith_sign) + integer)).setParseAction(lambda t: float(t[0]))
int_num = Combine(Optional(arith_sign) + integer + Optional(exponential_marker + Optional('+') + integer)).setParseAction(lambda t: int(t[0]))
def parse_date_str(token):
y, _, m, _, d = token
try:
date_object = date(*map(int, (y, m, d)))
except ValueError:
raise QueryException('Date %s-%s-%s is out of range' % (y, m, d))
return date_object
date_str = (integer('year') + '-' + integer('month') + '-' + integer('day')).setParseAction(parse_date_str)
# define query tokens
identifier = Word(alphas, alphanums + '_$-').setName('identifier')
variable_name = delimitedList(identifier, '.', combine=True)
value_string = date_str | real_num | int_num | variable_name | quoted_string_excluding_quotes
# Define a where expression
where_expression = Forward()
binary_operator_statement = (variable_name + binary_op + value_string).setParseAction(self.binary_op_as_q)
free_text_statement = quotedString.copy().setParseAction(self.freetext_as_q)
operator_statement = binary_operator_statement | free_text_statement
where_condition = Group(operator_statement | ('(' + where_expression + ')'))
where_expression << where_condition + ZeroOrMore((and_ | or_) + where_expression)
# define the full grammar
query_statement = Forward()
query_statement << Group(where_expression).setResultsName("where")
return query_statement
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import quotedString, restOfLine, Keyword, nestedExpr, OneOrMore, Word, Literal, removeQuotes
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
def np(words, fn = gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME,), action=lambda x: self._set_name(x[2]))
for key in self.NULL_KEYS:
if key == id: continue
gr_line = gr_line | np((key,), action=self._parse_setter(key))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Console: ' + self[NAME]
def nonlocalDimensionAccessForComponents(components, codeBlock):
"""
Find all places in the `codeBlock` where any of `components` are accessed with
non-locally (usually integer-valued dimensions) and return a ``(componentName, resultDict, codeSlice)``
tuple for each such occurrence. The companion of `nonlocalDimensionAccessForVectors` and
to be used when `components` are components of vectors.
"""
# Optimise for the common case: if the code doesn't contain the string "=>", then we know it doesn't have any nonlocal access
if "=>" not in codeBlock.codeString:
return []
dictionaryElement = identifier + Suppress('=>') + sliceFor(Group(baseExpr))
nonlocalAccessDictParser = Dict(
ZeroOrMore(Group(dictionaryElement + Suppress(','))) + Group(dictionaryElement)
)
parser = identifier.setResultsName('name') \
+ nestedExpr('(', ')', nonlocalAccessDictParser, ignoreExpr).setResultsName('access')
parser.ignore(cppStyleComment.copy())
parser.ignore(quotedString.copy())
results = []
for tokens, start, end in parser.scanString(codeBlock.codeString):
if tokens.name not in components: continue
accessDict = {}
tokenDict = tokens.access[0].asDict()
for key, value in tokenDict.items():
accessDict[key] = (' '.join(value[0].asList()), value.slice.start)
results.append((tokens.name, accessDict, slice(start, end)))
return results
def parse(self, header):
comment = self._comment()
quoted = quotedString.copy().setParseAction(removeQuotes)
string = quoted | Word(printables, excludeChars='{},%')
enum_value = quotedString | Word(printables, excludeChars='{},%')
relation = (Suppress(CaselessLiteral("@relation")) +
Optional(restOfLine, default='default_name')('rel_name').setParseAction(lambda t: t.rel_name.strip()))
relation_part = ZeroOrMore(comment) + relation + ZeroOrMore(comment)
nominal = (Empty().copy().setParseAction(lambda t: self.ENUM) +
Suppress(Literal("{")) +
Group(delimitedList(enum_value, delim=self._separator))("next_arg").setParseAction(self.get_values) +
Suppress(Literal("}")))
date = CaselessLiteral("date") + Optional(CharsNotIn("{},\n"))("next_arg").setParseAction(self._adapt_date_format)
attributes_part = Forward()
relational = CaselessLiteral("relational") + attributes_part + Suppress(CaselessLiteral("@end")) + string
attr_type = (CaselessLiteral("numeric") | CaselessLiteral("string") | nominal | date | relational)("attr_type")
attribute = Suppress(CaselessLiteral("@attribute")) + (string.copy())("attr_name") + attr_type
attribute_line = comment | attribute
attributes_part << (Group(OneOrMore(attribute_line)))("children")
data_part = (CaselessLiteral("@data"))("data_start").setParseAction(lambda s, p, k: (lineno(p, s)))
arff_header = relation_part + attributes_part + data_part
attribute.setParseAction(self._create_attribute)
try:
result = arff_header.parseString(header, parseAll=True)
except ParseException as e:
raise HeaderError(FileType.ARFF, e.lineno, e.col, e.line, e)
self._relation_name = result.rel_name
self._find_relational(result.children)
self._linearize_attrs(result.children)
self._data_start = result.data_start
self._index = 0
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
def np(words, fn = gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME,), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np(PList('sd port'), gr_number, action=self._parse_setter(SDPORT))
gr_line = gr_line | np((ADDRESS,), action=self._parse_setter(ADDRESS))
gr_line = gr_line | np((PASSWORD,), action=self._parse_setter(PASSWORD))
gr_line = gr_line | np((DEVICE,), action=self._parse_setter(DEVICE))
gr_line = gr_line | np(PList('media type'), action=self._parse_setter(MEDIATYPE))
gr_line = gr_line | np(PList('auto changer'), gr_yn, action=self._parse_setter(AUTOCHANGER))
gr_line = gr_line | np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(MAXIMUMCONCURRENTJOBS))
gr_line = gr_line | np(PList('allow compression'), gr_yn, action=self._parse_setter(ALLOWCOMPRESSION))
gr_line = gr_line | np(PList('heartbeat interval'), action=self._parse_setter(HEARTBEATINTERVAL))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Storage: ' + self[NAME]
def _relational_delim_list(self, sep):
quoted = quotedString.copy()
quoted.setParseAction(lambda s, p, t: self._parse_rel(removeQuotes(s, p, t), sep))
value = quoted | Word(printables, excludeChars=('%' + sep))
value.setParseAction(self._inc_index)
comment = self._comment()
parser = comment | delimitedList(value, delim=sep)
return parser
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import quotedString, restOfLine, Keyword, nestedExpr, OneOrMore, Word, Literal, removeQuotes, nums, replaceWith, printables
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
def _handle_ip(*x):
a,b,c = x[2]
return ' %s = { %s }' % (a,c[0])
def _handle_fdaddr(*x):
a,b,c = x[2]
self._set(FDADDRESSES, ' %s' % '\n '.join(c))
return
def np(words, fn = gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME,), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np((ADDRESS,), action=self._parse_setter(ADDRESS))
gr_line = gr_line | np((CATALOG,), action=self._parse_setter(CATALOG_ID, dereference=True))
gr_line = gr_line | np((PASSWORD,), action=self._parse_setter(PASSWORD))
gr_line = gr_line | np(PList('file retention'), action=self._parse_setter(FILERETENTION))
gr_line = gr_line | np(PList('job retention'), action=self._parse_setter(JOBRETENTION))
gr_line = gr_line | np((PRIORITY,), gr_number, action=self._parse_setter(PRIORITY))
gr_line = gr_line | np(PList('working directory'), action=self._parse_setter(WORKINGDIRECTORY))
gr_line = gr_line | np(PList('pid directory'), action=self._parse_setter(PIDDIRECTORY))
gr_line = gr_line | np(PList('heart beat interval'), action=self._parse_setter(HEARTBEATINTERVAL))
gr_line = gr_line | np(PList('fd address'), action=self._parse_setter(FDADDRESS))
gr_line = gr_line | np(PList('fd source address'), action=self._parse_setter(FDSOURCEADDRESS))
gr_line = gr_line | np(PList('pki key pair'), action=self._parse_setter(PKIKEYPAIR))
gr_line = gr_line | np(PList('pki master key'), action=self._parse_setter(PKIMASTERKEY))
gr_line = gr_line | np(PList('fd port'), gr_number, action=self._parse_setter(FDPORT))
gr_line = gr_line | np(PList('auto prune'), gr_yn, action=self._parse_setter(AUTOPRUNE))
gr_line = gr_line | np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(FDPORT))
gr_line = gr_line | np(PList('pki encryption'), gr_yn, action=self._parse_setter(PKIENCRYPTION))
gr_line = gr_line | np(PList('pki signatures'), gr_yn, action=self._parse_setter(PKISIGNATURES))
# This is a complicated one
da_addr = np(('Addr','Port'), Word(printables), lambda x,y,z: ' '.join(z))
da_ip = np(('IPv4','IPv6','IP'), nestedExpr('{','}', OneOrMore(da_addr).setParseAction(lambda x,y,z: ' ; '.join(z)))).setParseAction(_handle_ip)
da_addresses = np(('fd addresses', FDADDRESSES), nestedExpr('{','}', OneOrMore(da_ip)), _handle_fdaddr)
gr_res = OneOrMore(gr_line|da_addresses)
result = gr_res.parseString(string, parseAll=True)
return 'Client: ' + self[NAME]
def _relational_delim_list(self, sep):
quoted = quotedString.copy()
quoted.setParseAction(
lambda s, p, t: self._parse_rel(removeQuotes(s, p, t), sep))
value = quoted | Word(printables, excludeChars=('%' + sep))
value.setParseAction(self._inc_index)
comment = self._comment()
parser = comment | delimitedList(value, delim=sep)
return parser
def _create_grammar(self):
"""
Pyparsing implementation of a where clause grammar based on http://pyparsing.wikispaces.com/file/view/simpleSQL.py
The query language is a series of statements separated by AND or OR operators and parentheses can be used to group/provide
precedence.
A statement is a combination of three strings "<filter> <operator> <value>" or "<filter> <operator> <filter>".
A value can be a string, integer or a real(floating) number or a (ISO YYYY-MM-DD) date.
An operator must be one of "= != < > >= <= !:" and are translated into django __lte or equivalent suffixes.
See self.as_q
Example
something < 10 AND other >= 2015-01-01 AND (foo < 1 OR bar > 1)
"""
quoted_string_excluding_quotes = QuotedString(
'"',
escChar='\\').setParseAction(lambda token: StringValue(token[0]))
and_ = Keyword('and', caseless=True)
or_ = Keyword('or', caseless=True)
binary_op = oneOf('=> =< = < > >= <= : != !:',
caseless=True).setResultsName('operator')
# define query tokens
identifier = Word(alphas, alphanums + '_$-.').setName('identifier')
raw_value_chars = alphanums + '_$-+/$%*;?@[]\\^`{}|~.'
raw_value = Word(raw_value_chars, raw_value_chars).setName('raw_value')
value_string = quoted_string_excluding_quotes | raw_value
# Define a where expression
where_expression = Forward()
binary_operator_statement = (identifier + binary_op +
value_string).setParseAction(
self._binary_op_to_q)
unary_operator_statement = (identifier |
(Char('!') + identifier)).setParseAction(
self._unary_op_to_q)
free_text_statement = quotedString.copy().setParseAction(
self._freetext_to_q)
operator_statement = binary_operator_statement | free_text_statement | unary_operator_statement
where_condition = Group(operator_statement
| ('(' + where_expression + ')'))
where_expression << where_condition + ZeroOrMore(
(and_ | or_) + where_expression)
# define the full grammar
query_statement = Forward()
query_statement << Group(where_expression).setResultsName("where")
return query_statement
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_name = Keyword('name', caseless=True) + gr_eq + gr_opt_quoted_string
gr_name.setParseAction(lambda x, y=self: y._set_name(x[2]))
gr_yn = Keyword('yes', caseless=True).setParseAction(
replaceWith('1')) | Keyword('no', caseless=True).setParseAction(
replaceWith('0'))
gr_phrase = Group(
OneOrMore(gr_stripped_string | Word(alphanums)) + gr_eq +
gr_opt_quoted_string)
def np(words, fn=gr_opt_quoted_string, action=print):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_ifsc = np(PList('Ignore File Set Changes'),
gr_yn,
action=self._parse_setter(IGNORECHANGES))
gr_evss = np(PList('Enable VSS'),
gr_yn,
action=self._parse_setter(VSSENABLED))
gr_i_option = Group(
Keyword(OPTIONS, caseless=True) +
nestedExpr('{', '}', Regex('[^\}]+', re.MULTILINE)))
gr_e_option = gr_i_option.copy()
gr_i_file = gr_phrase.copy()
gr_e_file = gr_phrase.copy()
gr_inc = Keyword('include', caseless=True) + nestedExpr(
'{', '}', OneOrMore(gr_i_option | gr_i_file))
gr_inc.addParseAction(self._parse_add_entry)
gr_exc = Keyword('exclude', caseless=True) + nestedExpr(
'{', '}', OneOrMore(gr_e_option | gr_e_file))
gr_exc.addParseAction(self._parse_add_entry)
gr_res = OneOrMore(gr_name | gr_inc | gr_exc | gr_ifsc | gr_evss)
result = gr_res.parseString(string, parseAll=True)
return 'Fileset: ' + self[NAME]
def substitute_date(self):
def action(s, loc, tokens):
try:
return self.parser.get_time_stamp(removeQuotes(s, loc, tokens))
except ValueError as e:
raise DateValFormatError(e)
DATEXPR = quotedString.copy()
EXPR = boolexpr(VAL=DATEXPR)
DATEXPR.setParseAction(action)
try:
self._expr_pattern = EXPR.parseString(self._expr_pattern, parseAll=True)[0]
except ParseException as e:
raise DateSyntaxError(e.lineno, e.col, e.line, e)
def rhs_value_p():
Ipv4Address = Combine(Word(nums) +
('.' + Word(nums)) * 3).setResultsName('ipv4')
Ipv4Address = Ipv4Address.setParseAction(lambda s, l, toks: toks[0])
Int = Word(nums)
Int = Int.setParseAction(lambda s, l, toks: int(toks[0]))
Float = Combine(Word(nums) + '.' + Word(nums)).setResultsName('float')
Float = Float.setParseAction(lambda s, l, toks: float(toks[0]))
String = quotedString.copy().addParseAction(pyparsing.removeQuotes)
rhs = pyparsing.Or([String, Int, Float, Ipv4Address])
return rhs
def substitute_date(self):
def action(s, loc, tokens):
try:
return self.parser.get_time_stamp(removeQuotes(s, loc, tokens))
except ValueError as e:
raise DateValFormatError(e)
DATEXPR = quotedString.copy()
EXPR = boolexpr(VAL=DATEXPR)
DATEXPR.setParseAction(action)
try:
self._expr_pattern = EXPR.parseString(self._expr_pattern,
parseAll=True)[0]
except ParseException as e:
raise DateSyntaxError(e.lineno, e.col, e.line, e)
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(
replaceWith('1')) | Keyword('no', caseless=True).setParseAction(
replaceWith('0'))
def np(words, fn=gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME, ), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np(
PList('sd port'), gr_number, action=self._parse_setter(SDPORT))
gr_line = gr_line | np((ADDRESS, ), action=self._parse_setter(ADDRESS))
gr_line = gr_line | np(
(PASSWORD, ), action=self._parse_setter(PASSWORD))
gr_line = gr_line | np((DEVICE, ), action=self._parse_setter(DEVICE))
gr_line = gr_line | np(PList('media type'),
action=self._parse_setter(MEDIATYPE))
gr_line = gr_line | np(PList('auto changer'),
gr_yn,
action=self._parse_setter(AUTOCHANGER))
gr_line = gr_line | np(
PList('maximum concurrent jobs'),
gr_number,
action=self._parse_setter(MAXIMUMCONCURRENTJOBS))
gr_line = gr_line | np(PList('allow compression'),
gr_yn,
action=self._parse_setter(ALLOWCOMPRESSION))
gr_line = gr_line | np(PList('heartbeat interval'),
action=self._parse_setter(HEARTBEATINTERVAL))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Storage: ' + self[NAME]
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
def np(words, fn = gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME,), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np(PList('pool type'), action=self._parse_setter(POOLTYPE))
gr_line = gr_line | np(PList('maximum volumes'), action=self._parse_setter(MAXIMUMVOLUMES))
gr_line = gr_line | np((STORAGE,), action=self._parse_setter(STORAGE))
gr_line = gr_line | np(PList('use volume once'), gr_yn, action=self._parse_setter(USEVOLUMEONCE))
gr_line = gr_line | np(PList('catalog files'), gr_yn, action=self._parse_setter(CATALOGFILES))
gr_line = gr_line | np(PList('auto prune'), gr_yn, action=self._parse_setter(AUTOPRUNE))
gr_line = gr_line | np((RECYCLE,), gr_yn, action=self._parse_setter(RECYCLE))
gr_line = gr_line | np(PList('recycle oldest volume'), gr_yn, action=self._parse_setter(RECYCLEOLDESTVOLUME))
gr_line = gr_line | np(PList('recycle current volume'), gr_yn, action=self._parse_setter(RECYCLECURRENTVOLUME))
gr_line = gr_line | np(PList('purge oldest volume'), gr_yn, action=self._parse_setter(PURGEOLDESTVOLUME))
gr_line = gr_line | np(PList('maximum volume jobs'), gr_number, action=self._parse_setter(MAXIMUMVOLUMEJOBS))
gr_line = gr_line | np(PList('maximum volume files'), gr_number, action=self._parse_setter(MAXIMUMVOLUMEFILES))
gr_line = gr_line | np(PList('maximum volume bytes'), action=self._parse_setter(MAXIMUMVOLUMEBYTES))
gr_line = gr_line | np(PList('volume use duration'), action=self._parse_setter(VOLUMEUSEDURATION))
gr_line = gr_line | np(PList('volume retention'), action=self._parse_setter(VOLUMERETENTION))
gr_line = gr_line | np(PList('action on purge'), action=self._parse_setter(ACTIONONPURGE))
gr_line = gr_line | np(PList('scratch pool'), action=self._parse_setter(SCRATCHPOOL))
gr_line = gr_line | np(PList('recycle pool'), action=self._parse_setter(RECYCLEPOOL))
gr_line = gr_line | np(PList('file retention'), action=self._parse_setter(FILERETENTION))
gr_line = gr_line | np(PList('job retention'), action=self._parse_setter(JOBRETENTION))
gr_line = gr_line | np(PList('cleaning prefix'), action=self._parse_setter(CLEANINGPREFIX))
gr_line = gr_line | np(PList('label format'), action=self._parse_setter(LABELFORMAT))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Pool: ' + self[NAME]
def targetComponentsForOperatorsInString(operatorNames, codeBlock):
"""
Return a list of pairs of operator names and their targets that are in `codeString`.
The valid operator names searched for are `operatorNames`. For example, if 'L' is in `operatorNames`,
then in the code ``L[phi]`` the return value would be ``('L', 'phi', slice(firstCharacterIndex, lastCharacterIndex))``.
"""
parser = MatchFirst(Keyword(operatorName) for operatorName in operatorNames).setResultsName('name') \
+ Optional(nestedExpr('[', ']', baseExpr, ignoreExpr).setResultsName('target'))
parser.ignore(cppStyleComment.copy())
parser.ignore(quotedString.copy())
results = []
for tokens, start, end in parser.scanString(codeBlock.codeString):
if 'target' in tokens:
results.append((tokens.name, ''.join(tokens.target.asList()[0]), slice(start, end)))
else:
raise CodeParserException(codeBlock, start, "Invalid use of '%s' operator in code block." % tokens.name)
return results
def parse(self, header):
comment = self._comment()
quoted = quotedString.copy().setParseAction(removeQuotes)
string = quoted | Word(printables, excludeChars='{},%')
enum_value = quotedString | Word(printables, excludeChars='{},%')
relation = (Suppress(CaselessLiteral("@relation")) +
Optional(restOfLine, default='default_name')
('rel_name').setParseAction(lambda t: t.rel_name.strip()))
relation_part = ZeroOrMore(comment) + relation + ZeroOrMore(comment)
nominal = (Empty().copy().setParseAction(lambda t: self.ENUM) +
Suppress(Literal("{")) +
Group(delimitedList(enum_value, delim=self._separator))
("next_arg").setParseAction(self.get_values) +
Suppress(Literal("}")))
date = CaselessLiteral("date") + Optional(
CharsNotIn("{},\n"))("next_arg").setParseAction(
self._adapt_date_format)
attributes_part = Forward()
relational = CaselessLiteral(
"relational") + attributes_part + Suppress(
CaselessLiteral("@end")) + string
attr_type = (CaselessLiteral("numeric") | CaselessLiteral("string")
| nominal | date | relational)("attr_type")
attribute = Suppress(CaselessLiteral("@attribute")) + (
string.copy())("attr_name") + attr_type
attribute_line = comment | attribute
attributes_part << (Group(OneOrMore(attribute_line)))("children")
data_part = (CaselessLiteral("@data"))("data_start").setParseAction(
lambda s, p, k: (lineno(p, s)))
arff_header = relation_part + attributes_part + data_part
attribute.setParseAction(self._create_attribute)
try:
result = arff_header.parseString(header, parseAll=True)
except ParseException as e:
raise HeaderError(FileType.ARFF, e.lineno, e.col, e.line, e)
self._relation_name = result.rel_name
self._find_relational(result.children)
self._linearize_attrs(result.children)
self._data_start = result.data_start
self._index = 0
def checkForIntegerDivision(codeBlock):
"""
Raise a CodeParserException if the code contains what looks like an integer division.
i.e. ``9/2`` or the like. This is because the user is likely to get unexpected results.
The most notorious example of this is ``1/2`` which evaluates to zero.
"""
parser = numericConstant.setResultsName('numerator') + '/' + numericConstant.setResultsName('denominator')
parser.ignore(cppStyleComment.copy())
parser.ignore(quotedString.copy())
for tokens, start, end in parser.scanString(codeBlock.codeString):
if tokens.numerator.isdigit() and tokens.denominator.isdigit():
raise CodeParserException(
codeBlock, start,
"It looks like you are trying to divide two integers.\n"
"One of the oddities of the C language is that the result of such an expression\n"
"is the floor of that division instead of the real value.\n"
"For example '1/2' would give '0' instead of '0.5'.\n"
"The way to fix this is to turn one or both of the integers into real numbers\n"
"by adding a decimal point. For example, '1/2' should be written as '1.0/2.0'.\n\n"
"If you feel this warning is given in error, send an email to [email protected]"
)
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_name = Keyword('name', caseless=True) + gr_eq + gr_opt_quoted_string
gr_name.setParseAction(lambda x, y=self: y._set_name(x[2]))
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
gr_phrase = Group(OneOrMore(gr_stripped_string | Word(alphanums)) + gr_eq + gr_opt_quoted_string)
def np(words, fn = gr_opt_quoted_string, action=print):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_ifsc = np(PList('Ignore File Set Changes'), gr_yn, action=self._parse_setter(IGNORECHANGES))
gr_evss = np(PList('Enable VSS'), gr_yn, action=self._parse_setter(VSSENABLED))
gr_i_option = Group(Keyword(OPTIONS, caseless=True) + nestedExpr('{','}', Regex('[^\}]+', re.MULTILINE)))
gr_e_option = gr_i_option.copy()
gr_i_file = gr_phrase.copy()
gr_e_file = gr_phrase.copy()
gr_inc = Keyword('include', caseless=True) + nestedExpr('{','}', OneOrMore(gr_i_option | gr_i_file))
gr_inc.addParseAction(self._parse_add_entry)
gr_exc = Keyword('exclude', caseless=True) + nestedExpr('{','}', OneOrMore(gr_e_option | gr_e_file))
gr_exc.addParseAction(self._parse_add_entry)
gr_res = OneOrMore(gr_name | gr_inc | gr_exc | gr_ifsc | gr_evss)
result = gr_res.parseString(string, parseAll=True)
return 'Fileset: ' + self[NAME]
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
def np(words, fn=gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME, ), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np(
(USER, 'dbuser', 'db user'), action=self._parse_setter(USER))
gr_line = gr_line | np((PASSWORD, 'dbpassword', 'db password'),
action=self._parse_setter(PASSWORD))
gr_line = gr_line | np(PList(DBSOCKET),
action=self._parse_setter(DBSOCKET))
gr_line = gr_line | np(
PList(DBPORT), gr_number, action=self._parse_setter(DBPORT))
gr_line = gr_line | np(PList('db name'),
action=self._parse_setter(DBNAME))
gr_line = gr_line | np(PList('db address'),
action=self._parse_setter(DBADDRESS))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Catalog: ' + self[NAME]
def _parser_piece_destination_and_title():
"""
Return PyParsing element to match the destination and title of a
markdown link.
"""
# Capture everything between the balanced parentheses
# Then parse it later.
dest_and_title = originalTextFor(nestedExpr(
opener="(",
closer=")")).addParseAction(lambda s, l, toks: toks[0][1:-1])
destination = Combine(
# Zero or more non-space characters.
# But before each character (exact=1) check if we have a
# shortcode. If we do, allow that.
ZeroOrMore(
originalTextFor(nestedExpr(opener=R"{{<", closer=">}}"))
| originalTextFor(nestedExpr(opener=R"{{%", closer="%}}"))
| CharsNotIn(" \t", exact=1))).setResultsName("destination")
# CommonMark requires link title to be encased in single-quotes,
# double-quotes, or wrapped in parentheses. Let's not bother with
# the parentheses case for now.
title = (quotedString.copy().setResultsName("title").setParseAction(
lambda s, l, toks: unescape_quoted_string(toks[0])))
# This will parse the contents of dest_and_title
dest_and_title_parser = destination + Optional(White(" ") +
title) + StringEnd()
def back_parse_action(_s, _l, toks):
return dest_and_title_parser.parseString(toks[0])
dest_and_title.addParseAction(back_parse_action)
return dest_and_title
def _delim_list(self, sep):
quoted = quotedString.copy()
value = quoted | Word(printables, excludeChars=('%' + sep))
parser = delimitedList(value, delim=sep)
return parser
def performIPOperatorSanityCheck(componentName, propagationDimension, operatorCodeSlice, codeBlock):
"""
Check that the user hasn't tried to use an IP operator where an IP operator cannot be used.
IP operators must be diagonal, so one cannot have expressions of the form ``dy_dt = L[x];`` for IP operators.
This is valid for EX operators, but not for IP. This is a common mistake for users to make, and so we should
do our best to spot it and report the error. Another mistake users make is trying to multiply the operator,
for example ``dy_dt = i*L[y];``. This code does a sophisticated validation by constructing a parse tree for
each statement in the code taking into account operator precedence. This sanity checking is even able to pick
up problems such as ``dphi_dt = i*(V*phi + U*mod2(phi)*phi + T[phi]);``.
If the user's code passes this test, then it is a reasonable assumption that they are using IP operators safely.
"""
operatorString = codeBlock.codeString[operatorCodeSlice]
expr = Forward()
operatorKeyword = Keyword(operatorString).setResultsName('targetOperator')
operand = operatorKeyword \
| (identifier + Group('(' + delimitedList(expr) + ')')) \
| (identifier + Group(OneOrMore('[' + expr + ']'))) \
| quotedString.copy() \
| identifier \
| numericConstant
operand.ignore(cppStyleComment.copy())
expr << operatorPrecedence(
operand,
[
(oneOf('++ --'), 1, opAssoc.LEFT),
(oneOf('. ->'), 2, opAssoc.LEFT),
(~oneOf('-> -= += *= &= |=') + oneOf('+ - ! ~ * & ++ --'), 1, opAssoc.RIGHT),
(~oneOf('*= /= %=') + oneOf('* / %'), 2, opAssoc.LEFT),
(~oneOf('++ -- -> -= +=') + oneOf('+ -'), 2, opAssoc.LEFT),
# Although the operators below don't all have the same precedence, as we don't actually
# care about them as they are all invalid uses of the IP operator, we can cheat and lump
# them together
(~oneOf('<<= >>= &= |=') + oneOf('<< >> < <= > >= == != & ^ | && ||'), 2, opAssoc.LEFT),
# Correct ordering
# (~oneOf('<<= >>=') + oneOf('<< >>'), 2, opAssoc.LEFT),
# (~oneOf('<< >> <<= >>=') + oneOf('< <= > >='), 2, opAssoc.LEFT),
# (oneOf('== !='), 2, opAssoc.LEFT),
# (~oneOf('&& &=') + '&', 2, opAssoc.LEFT),
# ('^', 2, opAssoc.LEFT),
# (~oneOf('|| |=') + '|', 2, opAssoc.LEFT),
# ('&&', 2, opAssoc.LEFT),
# ('||', 2, opAssoc.LEFT),
(('?',':'), 3, opAssoc.RIGHT),
(~Literal('==') + oneOf('= += -= *= /= %= <<= >>= &= ^= |= =>'), 2, opAssoc.RIGHT),
(',', 2, opAssoc.LEFT),
]
)
expr.ignore(cppStyleComment.copy())
statement = expr + Suppress(';')
stack = []
expectedAssignmentVariable = 'd%(componentName)s_d%(propagationDimension)s' % locals()
def validateStack():
"""
It is the job of this function to validate the operations that the located operator is involved in.
The stack describes the part of the parse tree in which the operator was found. The first element in the stack
is the outermost operation, and the last the innermost. The last element is guaranteed to be the operator itself.
"""
# Reverse the stack as we want to search the parse tree from inner-most expression to outer-most.
stack.reverse()
assignmentHit = False
errorMessageCommon = "Due to the way IP operators work, they can only contribute to the derivative of the variable " \
"they act on, i.e. dx_dt = L[x]; not dy_dt = L[x];\n\n"
# We don't need to check the first element of the stack
# as we are guaranteed that it is the operator itself. This will be useful for determining
# which part of the parse tree we should be looking at.
for idx, node in enumerate(stack[1:]):
if len(node) == 1: continue
# idx is the index in the stack of the next element *deeper* in the parse tree.
previousStackEntry = stack[idx]
if not isinstance(stack[idx], basestring):
previousStackEntry = previousStackEntry.asList()
binaryOpIdx = node.asList().index(previousStackEntry) - 1
if binaryOpIdx < 0: binaryOpIdx = 1
# Unary '+' is safe.
if node[0] == '+': continue
# Binary '+' is safe.
if node[binaryOpIdx] == '+': continue
# Binary '-' is safe if the operator is the first argument.
if node[binaryOpIdx] == '-' and node.asList().index(previousStackEntry) == 0: continue
# Assignment is safe if it there is only one, and if it's to the right variable
if node[binaryOpIdx] in ['=', '+=']:
if node[0] == expectedAssignmentVariable:
assignmentHit = True
continue
else:
return errorMessageCommon + "In this case, you should probably use an EX operator instead of an "\
"IP operator."
else:
return errorMessageCommon + "You appear to be using the IP operator in an unsafe operation. " \
"The most likely cause is trying to multiply it by something, e.g. dphi_dt = 0.5*L[phi]; "\
"If this is the cause and you are multiplying by a constant, just move the constant into the "\
"definition of the operator itself. i.e. L = -0.5*kx*kx; If you are multiplying by something "\
"that isn't constant e.g. dphi_dt = x*L[phi]; where x is a dimension, you must use an EX operator "\
"instead."
if not assignmentHit:
return errorMessageCommon + "You appear to be missing the assignment for this particular operator."
return True
class FoundTargetException(Exception): pass
def findOperatorInParseTree(results):
stack.append(results)
if 'targetOperator' in results:
stack.append(results.targetOperator)
raise FoundTargetException()
for item in results:
if isinstance(item, basestring): continue
findOperatorInParseTree(item)
del stack[-1]
try:
foundOperator = False
for tokens, start, end in statement.scanString(codeBlock.codeString):
if start > operatorCodeSlice.stop or end < operatorCodeSlice.start: continue
try:
findOperatorInParseTree(tokens)
except FoundTargetException:
foundOperator = True
result = validateStack()
if result is not True:
raise CodeParserException(
codeBlock,
operatorCodeSlice.start,
result + ("\n\nThe conflict was caused by the operator '%s'." \
% operatorString)
)
if not foundOperator:
parserWarning(
codeBlock.xmlElement,
"Unable to check the safety of your IP operator '%s' because the containing expression could not be found. "
"Please send a copy of your script to [email protected] so this problem can be investigated." \
% operatorString
)
except RuntimeError:
parserWarning(
codeBlock.xmlElement,
"Unable to check the safety of your IP operator because your code is too deeply nested."
)
def parser():
eq = Keyword("==").suppress()
ne = Keyword("!=").suppress()
gte = Keyword(">=").suppress()
lte = Keyword("<=").suppress()
gt = Keyword(">").suppress()
lt = Keyword("<").suppress()
lparen = Keyword("(").suppress()
rparen = Keyword(")").suppress()
dotdot = Keyword("..").suppress()
colon = Keyword(":").suppress()
assignOp = Keyword('=').suppress()
sign = Word("+-", exact=1)
nonzero = ''.join([str(i) for i in range(1, 10)])
True_ = Word("true").setParseAction(lambda *args: True)
False_ = Word("false").setParseAction(lambda *args: False)
Boolean_ = True_ | False_
Int_ = Combine(Optional(sign) + Word(nonzero, nums)).setParseAction(
lambda l, s, t: int(t[0]))
Hex_ = Regex("0x[A-Fa-f0-9]+").setResultsName('hex')
Term_ = Word(alphas, alphanums + "_")
VarToken_ = Combine('$' + Term_)
Var_ = Group(VarToken_).setResultsName('var')
String_ = quotedString.copy().setParseAction(removeQuotes).setResultsName(
'string')
Val_ = Forward()
Vals_ = Group(delimitedList(Val_, delim=' ')).setResultsName('vals')
In_ = Group(Val_ + Keyword("in", caseless=True).suppress() + lparen +
Vals_ + rparen).setResultsName('in')
Eq_ = Group(Val_ + eq + Val_).setResultsName('eq')
Ne_ = Group(Val_ + ne + Val_).setResultsName('ne')
Gte_ = Group(Val_ + gte + Val_).setResultsName('gte')
Lte_ = Group(Val_ + lte + Val_).setResultsName('lte')
Gt_ = Group(Val_ + gt + Val_).setResultsName('gt')
Lt_ = Group(Val_ + lt + Val_).setResultsName('lt')
Range_ = Group(lparen +
Optional(Val_, default=None).setResultsName('min') +
dotdot +
Optional(Val_, default=None).setResultsName('max') +
rparen).setResultsName('range')
Or_ = Forward()
Parens_ = Group(Suppress("(") + Or_ +
Suppress(")")).setResultsName("parens") | Val_
Not_ = Forward()
Not_ << (Group(Suppress(Keyword("not", caseless=True)) +
Not_).setResultsName("not") | Parens_)
And_ = Forward()
And_ << (Group(Not_ + Suppress(Keyword("and", caseless=True)) +
And_).setResultsName("and") | Not_)
Or_ << (Group(And_ + Suppress(Keyword("or", caseless=True)) +
Or_).setResultsName("or") | And_)
Assign_ = Group(VarToken_ + assignOp + Val_).setResultsName('assign')
Contract_ = Regex(
'([A-Za-z_][A-Za-z0-9_]*)'
'\(((?:0x[A-Fa-f0-9]+)'
'|(?:\$[A-Za-z_][A-Za-z0-9_]*))\)').setResultsName('contract')
Send_ = Group((Keyword("send") | Keyword("asend")) + Optional(Int_) +
(Contract_ | Var_) + Optional(Var_ | Term_) +
Optional(Vals_)).setResultsName('send')
Call_ = Group(
Keyword("call") + Optional(Int_) + (Var_ | Contract_) +
Optional(Var_ | Term_) + Optional(Vals_)).setResultsName('call')
Address_ = Group(Keyword("address") +
(Var_ | Contract_)).setResultsName('address')
Deploy_ = Group(Keyword("deploy") + (Var_ | Term_) +
(Var_ | Term_)).setResultsName('deploy')
Expression_ = (And_ | Or_ | Eq_ | Ne_ | Gte_
| Lte_ | Gt_ | Lt_ | In_ | Range_
| Call_ | Deploy_ | Send_)
Val_ << (Hex_ | Int_ | Boolean_ | String_ | Var_ | Expression_)
Statement_ = (Contract_ | Assign_ | Send_ | Call_ | Deploy_ | Val_)
return Statement_
def parse_string(self, string):
# {{{ boilerplate. Sigh
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
FTR: this is hideous.
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables, nullDebugAction
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))
def np(words, fn = gr_opt_quoted_string, action=nullDebugAction):
p = Keyword(words[0], caseless=True).setDebug(bacula_tools.DEBUG)
for w in words[1:]:
p = p | Keyword(w, caseless=True).setDebug(bacula_tools.DEBUG)
p = p + gr_eq + fn
p.setParseAction(action)
return p
# }}}
# {{{ # Easy ones that go don't take embedded spaces because I say so.
gr_line = np((NAME,), action=lambda x: self._set_name(x[2]))
for key in [TYPE, LEVEL, REPLACE, BASE, RUN, WHERE]:
gr_line = gr_line | np((key,), action=self._parse_setter(key))
# }}}
# {{{ # Group of _id variables
gr_line = gr_line | np(PList('differential pool'), action=self._parse_setter(DIFFERENTIALPOOL_ID))
gr_line = gr_line | np(PList('file set'), action=self._parse_setter(FILESET_ID, dereference=True))
gr_line = gr_line | np(PList('full pool'), action=self._parse_setter(FULLPOOL_ID))
gr_line = gr_line | np((CLIENT,), action=self._parse_setter(CLIENT_ID, dereference=True))
gr_line = gr_line | np(PList('incremental pool'), action=self._parse_setter(INCREMENTALPOOL_ID))
gr_line = gr_line | np((MESSAGES,), action=self._parse_setter(MESSAGES_ID, dereference=True))
gr_line = gr_line | np((POOL,), action=self._parse_setter(POOL_ID, dereference=True))
gr_line = gr_line | np((SCHEDULE,), action=self._parse_setter(SCHEDULE_ID, dereference=True))
gr_line = gr_line | np((STORAGE,), action=self._parse_setter(STORAGE_ID, dereference=True))
gr_line = gr_line | np(PList('job defs'), action=self._parse_setter(JOB_ID, dereference=True))
# }}}
# {{{ # INTs
gr_line = gr_line | np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(MAXIMUMCONCURRENTJOBS))
gr_line = gr_line | np(PList('re schedule times'), gr_number, action=self._parse_setter(RESCHEDULETIMES))
gr_line = gr_line | np((PRIORITY,), gr_number, action=self._parse_setter(PRIORITY))
# }}}
# {{{ # True/False
gr_line = gr_line | np((ACCURATE,), gr_yn, action=self._parse_setter(ACCURATE))
gr_line = gr_line | np(PList('allow duplicate jobs'), gr_yn, action=self._parse_setter(ALLOWDUPLICATEJOBS))
gr_line = gr_line | np(PList('allow mixed priority'), gr_yn, action=self._parse_setter(ALLOWMIXEDPRIORITY))
gr_line = gr_line | np(PList('cancel lower level duplicates'), gr_yn, action=self._parse_setter(CANCELLOWERLEVELDUPLICATES))
gr_line = gr_line | np(PList('cancel queued duplicates'), gr_yn, action=self._parse_setter(CANCELQUEUEDDUPLICATES))
gr_line = gr_line | np(PList('cancel running duplicates'), gr_yn, action=self._parse_setter(CANCELRUNNINGDUPLICATES))
gr_line = gr_line | np((ENABLED,), gr_yn, action=self._parse_setter(ENABLED))
gr_line = gr_line | np(PList('prefer mounted volumes'), gr_yn, action=self._parse_setter(PREFERMOUNTEDVOLUMES))
gr_line = gr_line | np(PList('prefix links'), gr_yn, action=self._parse_setter(PREFIXLINKS))
gr_line = gr_line | np(PList('prune files'), gr_yn, action=self._parse_setter(PRUNEFILES))
gr_line = gr_line | np(PList('prune jobs'), gr_yn, action=self._parse_setter(PRUNEJOBS))
gr_line = gr_line | np(PList('prune volumes'), gr_yn, action=self._parse_setter(PRUNEVOLUMES))
gr_line = gr_line | np(PList('re run failed levels'), gr_yn, action=self._parse_setter(RERUNFAILEDLEVELS))
gr_line = gr_line | np(PList('re schedule on error'), gr_yn, action=self._parse_setter(RESCHEDULEONERROR))
gr_line = gr_line | np(PList('spool attributes'), gr_yn, action=self._parse_setter(SPOOLATTRIBUTES))
gr_line = gr_line | np(PList('spool data'), gr_yn, action=self._parse_setter(SPOOLDATA))
gr_line = gr_line | np(PList('write boot strap'), gr_yn, action=self._parse_setter(WRITEPARTAFTERJOB))
# }}}
# {{{ # plain strings
gr_line = gr_line | np((NOTES,), action=self._parse_setter(NOTES))
gr_line = gr_line | np((ADDPREFIX, 'add prefix'), action=self._parse_setter(ADDPREFIX))
gr_line = gr_line | np((ADDSUFFIX, 'add suffix'), action=self._parse_setter(ADDSUFFIX))
gr_line = gr_line | np((BASE,), action=self._parse_setter(BASE))
gr_line = gr_line | np((BOOTSTRAP, 'boot strap'), action=self._parse_setter(BOOTSTRAP))
gr_line = gr_line | np((DIFFERENTIALMAXWAITTIME, 'differential max wait time', 'differentialmaxwait time', 'differentialmax waittime', 'differential maxwaittime', 'differentialmax wait time', 'differential maxwait time', 'differential max waittime'), action=self._parse_setter(DIFFERENTIALMAXWAITTIME))
gr_line = gr_line | np(('incremental-differentialmaxwaittime','incremental-differential maxwaittime','incremental-differentialmax waittime','incremental-differentialmaxwait time','incremental-differential max waittime','incremental-differential maxwait time','incremental-differentialmax wait time','incremental-differential max wait time',), action=self._parse_setter(IDMAXWAITTIME))
gr_line = gr_line | np((INCREMENTALMAXRUNTIME, 'incremental max run time', 'incrementalmaxrun time', 'incrementalmax runtime', 'incremental maxruntime', 'incrementalmax run time', 'incremental maxrun time', 'incremental max runtime'), action=self._parse_setter(INCREMENTALMAXRUNTIME))
gr_line = gr_line | np((MAXFULLINTERVAL, 'max full interval', 'max fullinterval', 'maxfull interval'), action=self._parse_setter(MAXFULLINTERVAL))
gr_line = gr_line | np((MAXIMUMBANDWIDTH, 'maximum band width', 'maximum bandwidth', 'maximumband width'), action=self._parse_setter(MAXIMUMBANDWIDTH))
gr_line = gr_line | np((MAXRUNSCHEDTIME, 'max run sched time', 'maxrunsched time', 'maxrun schedtime', 'max runschedtime', 'maxrun sched time', 'max runsched time', 'max run schedtime'), action=self._parse_setter(MAXRUNSCHEDTIME))
gr_line = gr_line | np((MAXRUNTIME, 'max run time', 'maxrun time', 'max runtime'), action=self._parse_setter(MAXRUNTIME))
gr_line = gr_line | np((MAXSTARTDELAY, 'max start delay', 'max startdelay', 'maxstart delay'), action=self._parse_setter(MAXSTARTDELAY))
gr_line = gr_line | np((MAXWAITTIME, 'max wait time', 'max waittime', 'maxwait time'), action=self._parse_setter(MAXWAITTIME))
gr_line = gr_line | np((REGEXWHERE, 'regex where'), action=self._parse_setter(REGEXWHERE))
gr_line = gr_line | np((RESCHEDULEINTERVAL, 're schedule interval', 're scheduleinterval', 'reschedule interval'), action=self._parse_setter(RESCHEDULEINTERVAL))
gr_line = gr_line | np((RUN,), action=self._parse_setter(RUN))
gr_line = gr_line | np((SPOOLSIZE, 'spool size'), action=self._parse_setter(SPOOLSIZE))
gr_line = gr_line | np((STRIPPREFIX, 'strip prefix'), action=self._parse_setter(STRIPPREFIX))
gr_line = gr_line | np((VERIFYJOB, 'verify job'), action=self._parse_setter(VERIFYJOB))
gr_line = gr_line | np((WHERE,), action=self._parse_setter(WHERE))
gr_line = gr_line | np((WRITEBOOTSTRAP, 'write boot strap', 'write bootstrap', 'writeboot strap'), action=self._parse_setter(WRITEBOOTSTRAP))
# }}}
# The ugliness that is run scripts
gr_line = gr_line | np(RBJ, gr_stripped_string,
action=self._parse_script(runsonclient=0, runswhen='Before'))
gr_line = gr_line | np(RAJ, gr_stripped_string,
action=self._parse_script(runsonclient=0, runswhen='After'))
gr_line = gr_line | np(RAFJ, gr_stripped_string,
action=self._parse_script(runsonsuccess=0, runsonfailure=1,
runsonclient=0, runswhen='After'))
gr_line = gr_line | np(CRBJ, gr_stripped_string,
action=self._parse_script(runswhen='Before'))
gr_line = gr_line | np(CRAJ, gr_stripped_string,
action=self._parse_script(runswhen='After'))
# This is a complicated one
gr_script_parts = np(('Command',), gr_stripped_string)
gr_script_parts = gr_script_parts | np((CONSOLE,), gr_stripped_string)
gr_script_parts = gr_script_parts | np(PList('Runs When'))
gr_script_parts = gr_script_parts | np(PList('Runs On Success'), gr_yn)
gr_script_parts = gr_script_parts | np(PList('Runs On Failure'), gr_yn)
gr_script_parts = gr_script_parts | np(PList('Runs On Client'), gr_yn)
gr_script_parts = gr_script_parts | np(PList('Fail Job On Error'), gr_yn)
gr_script = ((Keyword('Run Script', caseless=True) | Keyword('RunScript', caseless=True)) + nestedExpr('{','}', OneOrMore(gr_script_parts))).setParseAction(self._parse_script_full)
gr_res = OneOrMore(gr_line | gr_script)
try:
result = gr_res.parseString(string, parseAll=True)
except Exception as e:
print(e)
raise
return self.retlabel + ': '+ self[NAME]
def __parseInput(self,filename):
##----------------------------------------------------------------------
## Read input file
##----------------------------------------------------------------------
print " ... Parsing input file \"", filename ,"\""
sys.stdout.flush()
f = open(filename,'r')
expressions = f.read()
#print expressions
EQ,LBRACE,RBRACE,SEMI = map(Suppress,'={};')
LPAREN,RPAREN = map(Suppress,'()')
intEntry = Combine(Optional('-') + Word(nums)) \
.setParseAction(lambda t : int(t[0]))
realEntry = Combine(Optional('-') + Optional(Word(nums)) \
+ '.' + (Word(nums))) \
.setParseAction(lambda t : float(t[0]))
realEntry2 = Combine(Optional('-') + (Word(nums)) \
+ '.' + Optional(Word(nums))) \
.setParseAction(lambda t : float(t[0]))
numEntry = realEntry | realEntry2 | intEntry
sciEntry = Combine( numEntry \
+ (CaselessLiteral('E') \
+ Word('+-'+nums,nums))) \
.setParseAction(lambda t : float(t[0]))
realOrSciEntry = sciEntry | realEntry | realEntry2
# define tokens, expressions and entries
objEntry = Forward()
listEntry = Forward()
objListEntry = Forward()
flListEntry = Forward()
keyToken = Word(alphas+'_', alphanums+'_')
entryToken = ( keyToken | realOrSciEntry | intEntry |
quotedString.copy().setParseAction(removeQuotes))
# define lists
numList = OneOrMore(realOrSciEntry).setParseAction(lambda t : list([t[:]]))
intList = OneOrMore(intEntry).setParseAction(lambda t : list([t[:]]))
keyTokenList = OneOrMore(keyToken).setParseAction(lambda t : list([t[:]]))
entryList = numList | intList | keyTokenList
objExpr = ( Group(keyToken + objEntry)
| Group(keyToken + LBRACE + RBRACE))
objListExpr = Group(keyToken + objListEntry)
listExpr = ( Group(keyToken + listEntry) \
| Group(keyToken + LPAREN + RPAREN))
expr = Group(keyToken + entryToken + SEMI) \
| listExpr + SEMI
# expr needs to come before the list ones
mixedExpr = (objExpr | expr | objListExpr | listExpr)
objListEntry << ( LPAREN + Dict(OneOrMore(objExpr)) + RPAREN)
listEntry << ( LPAREN + entryList + RPAREN)
#| ( LPAREN + OneOrMore(listExpr) + RPAREN))# this last line allows nesting
objEntry << Group(LBRACE + Dict(OneOrMore(mixedExpr)) + RBRACE)
caseDict = Dict(OneOrMore(mixedExpr)) \
.ignore(cStyleComment).ignore('//' + restOfLine) \
.parseString(expressions,parseAll=True)
print ' Done'
sys.stdout.flush()
#print ' ... Input file dictionary:'
#print caseDict.dump()
#sys.stdout.flush()
f.close()
##----------------------------------------------------------------------
## shorter parsing code (too flexible)
##----------------------------------------------------------------------
#print 'NEW PARSING'
#f = open('zpyfInput','r')
#expressions = f.read()
#value = Forward()
#result = Forward()
#testDict = nestedExpr('{','}',result)
##testDict =
## Group (LBRACE + Optional(Dict(delimitedList(result,delim=SEMI)))
## + Optional(SEMI) + RBRACE)
#testList = Group (LPAREN + Optional(delimitedList(result,delim=SEMI))
# + Optional(SEMI) + RPAREN)
#testList2 = Group (LPAREN + Optional(OneOrMore(entryList))
# + Optional(SEMI) + RPAREN)
#result << Group ((keyToken | intEntry) + value )
#value << (entryToken + SEMI | testDict | testList | testList2 )
#tempD = Dict(delimitedList(result,delim=SEMI)) \
# .ignore(cStyleComment).ignore('//' + restOfLine) \
# .parseString(expressions)
#print tempD.dump()
#f.close()
return caseDict
def parse_imp(input):
# parse a string into an element of the abstract representation
# Grammar:
#
# <expr> ::= <integer>
# true
# false
# <identifier>
# ( if <expr> <expr> <expr> )
# ( function ( <name ... ) <expr> )
# ( <expr> <expr> ... )
#
# <decl> ::= var name = expr ;
#
# <stmt> ::= if <expr> <stmt> else <stmt>
# while <expr> <stmt>
# name <- <expr> ;
# print <expr> ;
# <block>
#
# <block> ::= { <decl> ... <stmt> ... }
#
# <toplevel> ::= <decl>
# <stmt>
#
idChars = alphas + "_+*-/?!=<>"
pIDENTIFIER = Word(idChars, idChars + "0123456789")
#### NOTE THE DIFFERENCE
pIDENTIFIER.setParseAction(
lambda result: EPrimCall(oper_deref, [EId(result[0])]))
pIDENTIFIERS = ZeroOrMore(pIDENTIFIER)
pIDENTIFIERS.setParseAction(lambda result: [result])
# A name is like an identifier but it does not return an EId...
pNAME = Word(idChars, idChars + "0123456789")
pNAMES = ZeroOrMore(pNAME)
pNAMES.setParseAction(lambda result: [result])
pINTEGER = Word("0123456789")
pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0]))))
pBOOLEAN = Keyword("true") | Keyword("false")
pBOOLEAN.setParseAction(
lambda result: EValue(VBoolean(result[0] == "true")))
def escapeString(inStr):
inStr = inStr[1:-1]
outStr = ""
i = 0
while (i < len(inStr) - 1):
if (inStr[i] == '\\'):
if (inStr[i + 1] == '\\' or inStr[i + 1] == '\"'):
i += 1
outStr += inStr[i]
i += 1
outStr += inStr[-1]
return outStr
pSTRING = quotedString.copy()
pSTRING.setParseAction(
lambda result: EValue(VString(escapeString(result[0]))))
pEXPR = Forward()
pEXPRS = ZeroOrMore(pEXPR)
pEXPRS.setParseAction(lambda result: [result])
pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")"
pIF.setParseAction(lambda result: EIf(result[2], result[3], result[4]))
def mkFunBody(params, body):
bindings = [(p, ERefCell(EId(p))) for p in params]
return ELet(bindings, body)
pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")"
pFUN.setParseAction(
lambda result: EFunction(result[3], mkFunBody(result[3], result[5])))
pBINDING = "(" + pNAME + pEXPR + ")"
pBINDING.setParseAction(lambda result: (result[1], result[2]))
pBINDINGS = ZeroOrMore(pBINDING)
pBINDINGS.setParseAction(lambda result: [result])
pCALL = "(" + pEXPR + pEXPRS + ")"
pCALL.setParseAction(lambda result: ECall(result[1], result[2]))
pEXPR << (pINTEGER | pBOOLEAN | pSTRING | pIDENTIFIER | pIF | pFUN | pCALL)
pDECL_VAR = "var" + pNAME + "=" + pEXPR + ";"
pDECL_VAR.setParseAction(lambda result: (result[1], result[3]))
pDECL_ARRAY = "var" + pNAME + "<-" + "(" + "new-array" + pEXPR + ")" + ";"
pDECL_ARRAY.setParseAction(lambda result: (result[1], EArray(result[5])))
# hack to get pDECL to match only PDECL_VAR (but still leave room
# to add to pDECL later)
pDECL = (pDECL_VAR | pDECL_ARRAY | NoMatch())
pDECLS = ZeroOrMore(pDECL)
pDECLS.setParseAction(lambda result: [result])
pSTMT = Forward()
pSTMT_IF_1 = "if" + pEXPR + pSTMT + "else" + pSTMT
pSTMT_IF_1.setParseAction(
lambda result: EIf(result[1], result[2], result[4]))
pSTMT_IF_2 = "if" + pEXPR + pSTMT
pSTMT_IF_2.setParseAction(
lambda result: EIf(result[1], result[2], EValue(VBoolean(True))))
pSTMT_WHILE = "while" + pEXPR + pSTMT
pSTMT_WHILE.setParseAction(lambda result: EWhile(result[1], result[2]))
pSTMT_PRINT = "print" + pEXPR + ";"
pSTMT_PRINT.setParseAction(
lambda result: EPrimCall(oper_print, [result[1]]))
pSTMT_UPDATE = pNAME + "<-" + pEXPR + ";"
pSTMT_UPDATE.setParseAction(
lambda result: EPrimCall(oper_update, [EId(result[0]), result[2]]))
pSTMTARR_UPDATE = pNAME + "[" + pEXPR + "]" + "<-" + pEXPR + ";"
pSTMTARR_UPDATE.setParseAction(lambda result: EPrimCall(
oper_update_arr, [EId(result[0]), result[5], result[2]]))
# pSTMT_FOR = "for" + "(" + pSTMT_UPDATE + pEXPR + ";" + pSTMT_UPDATE + ")" + pSTMT
# pSTMT_FOR.setParseAction(lambda result: EDo([result[2],EWhile(result[3], EDo([result[7],result[5]])) ] ))
pSTMT_FOR = "for" + pSTMT_UPDATE + pEXPR + ";" + pSTMT_UPDATE + pSTMT
pSTMT_FOR.setParseAction(lambda result: EDo(
[result[1], EWhile(result[2], EDo([result[5], result[4]]))]))
pSTMTS = ZeroOrMore(pSTMT)
pSTMTS.setParseAction(lambda result: [result])
def mkBlock(decls, stmts):
bindings = [(n, ERefCell(expr)) for (n, expr) in decls]
return ELet(bindings, EDo(stmts))
pSTMT_BLOCK = "{" + pDECLS + pSTMTS + "}"
pSTMT_BLOCK.setParseAction(lambda result: mkBlock(result[1], result[2]))
pDEFPROC = "procedure" + pNAME + "(" + pNAMES + ")" + pSTMT
pDEFPROC.setParseAction(
lambda result: {
"result":
"procedure",
"proc":
(result[1], EProcedure(result[3], mkFunBody(result[3], result[5])))
})
pSTMT_PROC = pIDENTIFIER + "(" + pEXPRS + ")" + ";"
pSTMT_PROC.setParseAction(lambda result: EProcCall(result[0], result[2]))
pWITH = "(" + Keyword(
"with") + pIDENTIFIER + pIDENTIFIER + "(" + pEXPRS + ")" + ")"
pWITH.setParseAction(lambda result: EWith(result[2], result[3], result[5]))
pNOTIMPLEMENTED = Keyword("<>")
pNOTIMPLEMENTED.setParseAction(lambda result: ENotImplemented())
pSTMT << (pSTMT_IF_1 | pSTMT_IF_2 | pSTMT_WHILE | pSTMT_PRINT | pSTMT_FOR
| pSTMT_UPDATE | pSTMTARR_UPDATE | pSTMT_BLOCK | pSTMT_PROC
| pWITH)
# can't attach a parse action to pSTMT because of recursion, so let's duplicate the parser
pTOP_STMT = pSTMT.copy()
pTOP_STMT.setParseAction(lambda result: {
"result": "statement",
"stmt": result[0]
})
pTOP_DECL = pDECL.copy()
pTOP_DECL.setParseAction(lambda result: {
"result": "declaration",
"decl": result[0]
})
pABSTRACT = "#abs" + pSTMT
pABSTRACT.setParseAction(lambda result: {
"result": "abstract",
"stmt": result[1]
})
pQUIT = Keyword("#quit")
pQUIT.setParseAction(lambda result: {"result": "quit"})
pDEFFUN = "(" + pNAME + "(" + pNAMES + ")" + pSTMT + ")"
pDEFFUN.setParseAction(lambda result: (result[
1], EProcedure(result[3], mkFunBody(result[3], result[5]))))
pDEFABSFUN = "(" + pNAME + "(" + pNAMES + ")" + pNOTIMPLEMENTED + ")"
pDEFABSFUN.setParseAction(lambda result: (result[1], ENotImplemented()))
pDEFFUNS = ZeroOrMore((pDEFFUN | pDEFABSFUN))
pDEFFUNS.setParseAction(lambda result: [result])
pTEMPLATE = Keyword(
"class"
) + "(" + pNAME + pIDENTIFIER + "(" + pNAMES + ")" + "(" + pIDENTIFIERS + ")" + "(" + pDEFFUNS + ")" + ")"
pTEMPLATE.setParseAction(
lambda result: {
"result":
"template",
"temp": (result[2],
ETemplate(False, result[2], result[3], result[5], result[
8], result[11]))
})
pABSTEMPLATE = Keyword(
"absclass"
) + "(" + pNAME + pIDENTIFIER + "(" + pNAMES + ")" + "(" + pIDENTIFIERS + ")" + "(" + pDEFFUNS + ")" + ")"
pABSTEMPLATE.setParseAction(
lambda result: {
"result":
"template",
"temp": (result[2],
ETemplate(True, result[2], result[3], result[5], result[
8], result[11]))
})
pNEWOBJ = Keyword("new") + pIDENTIFIER + "(" + pEXPRS + ")"
pNEWOBJ.setParseAction(lambda result: EObject(result[1], result[3]))
pOBJASS = Keyword("obj") + pIDENTIFIER + pNAME + "=" + pNEWOBJ
pOBJASS.setParseAction(
lambda result: {
"result": "objectassignment",
"assignment": (result[2], EObjectBinding(result[1], result[4]))
})
pMULTI = Keyword("#multi")
pMULTI.setParseAction(lambda result: {"result": "multi"})
pTOP = (pQUIT | pABSTRACT | pTOP_DECL | pTOP_STMT | pDEFPROC | pTEMPLATE
| pOBJASS | pMULTI | pABSTEMPLATE)
result = pTOP.parseString(input)[0]
return result # the first element of the result is the expression
This class determines the line in the original script that
corresponds to the part of the code block that triggered the
exception.
"""
def __init__(self, codeBlock, codeIndex, msg):
ParserException.__init__(self, codeBlock.xmlElement, msg)
self.columnNumber = col(codeIndex, codeBlock.codeString)
self.lineNumber = codeBlock.scriptLineNumber + lineno(codeIndex, codeBlock.codeString)-1
identifier = Word(alphas + '_', alphanums + '_')
numericConstant = Regex(r'\b((0(x|X)[0-9a-fA-F]*)|(([0-9]+\.?[0-9]*)|(\.[0-9]+))((e|E)(\+|-)?[0-9]+)?)(L|l|UL|ul|u|U|F|f|ll|LL|ull|ULL)?\b')
ignoreExpr = cppStyleComment.copy() | quotedString.copy()
baseExpr = Forward()
arrayAccess = originalTextFor(nestedExpr('[', ']', baseExpr, ignoreExpr))
parenthisedExpression = originalTextFor(nestedExpr('(', ')', baseExpr, ignoreExpr))
functionCall = nestedExpr('(', ')', delimitedList(baseExpr), ignoreExpr)
alphaNumPlusSafePunctuation = alphanums + '!#$%&\\*+-./:;<=>@^_`{|}~'
baseExpr << OneOrMore(originalTextFor(identifier + functionCall) | quotedString.copy() \
| identifier | numericConstant | arrayAccess | parenthisedExpression \
| Word(alphaNumPlusSafePunctuation))
baseExpr.ignore(cppStyleComment.copy())
def targetComponentsForOperatorsInString(operatorNames, codeBlock):
def __parseBCs(self,filenameBC):
fB = open(filenameBC,'r')
expressions = fB.read();
EQ,LBRACE,RBRACE,SEMI = map(Suppress,"={};")
LPAREN,RPAREN = map(Suppress,'()')
intEntry = Combine(Optional('-') + Word(nums)) \
.setParseAction(lambda t : int(t[0]))
realEntry = Combine(Optional('-') + Optional(Word(nums)) \
+ '.' + (Word(nums))) \
.setParseAction(lambda t : float(t[0]))
realEntry2 = Combine(Optional('-') + (Word(nums)) \
+ '.' + Optional(Word(nums))) \
.setParseAction(lambda t : float(t[0]))
numEntry = realEntry | realEntry2 | intEntry
sciEntry = Combine( numEntry \
+ (CaselessLiteral('E') \
+ Word('+-'+nums,nums))) \
.setParseAction(lambda t : float(t[0]))
realOrSciEntry = sciEntry | realEntry | realEntry2
# define tokens, expressions and entries
bcObjEntry = Forward()
bcObjListEntry = Forward()
#listEntry = Forward()
keyToken = Word(alphas+"_", alphanums+"_")
entryToken = ( keyToken | realOrSciEntry | intEntry |
quotedString.copy().setParseAction(removeQuotes))
# define lists
#numList = Group(OneOrMore(realOrSciEntry))
#intList = Group(OneOrMore(intEntry))
#keyTokenList = Group(OneOrMore(keyToken))
numList = (OneOrMore(realOrSciEntry)).setParseAction(lambda t : list([t[:]]))
intList = (OneOrMore(intEntry)).setParseAction(lambda t : list([t[:]]))
keyTokenList = (OneOrMore(keyToken)).setParseAction(lambda t : list([t[:]]))
entryList = numList | intList | keyTokenList
bcObjExpr = Group(keyToken + bcObjEntry) \
| Group (keyToken + LBRACE + RBRACE)
bcObjListExpr = Group(keyToken + bcObjListEntry)
#listExpr = ( Group(keyToken + listEntry) \
# | Group(keyToken + LPAREN + RPAREN))
expr = Group(keyToken + entryToken + SEMI) \
| Group(keyToken + LPAREN + entryList + RPAREN + SEMI)
# | listExpr + SEMI
# expr needs to come before the list ones
#bcMixedExpr = (bcObjExpr | expr | listExpr)
bcMixedExpr = (bcObjExpr | expr | bcObjListExpr)
bcObjListEntry << ( LPAREN + Dict(OneOrMore(bcObjExpr)) + RPAREN)
#listEntry << ( LPAREN + entryList + RPAREN)
# | ( LPAREN + OneOrMore(listExpr) + RPAREN))# this last line allows nesting
bcObjEntry << ( LBRACE + Dict(OneOrMore(bcMixedExpr)) + RBRACE)
#print expressions
bcDict_t = Dict(OneOrMore(bcMixedExpr)) \
.ignore(cStyleComment).ignore(cppStyleComment)\
.parseString(expressions,parseAll=True)
fB.close()
#print bcDict_t.dump()
#bcTypeList = []
#for i in range(len(msh.patchList)):
# #print bcDict_t[msh.patchList[i][0]]
# bcTypeList.append(bcDict_t[msh.patchList[i][0]].keys())
#bcDict = {}
#for iB in bcDict_t.keys():
# subBCDict = {}
# for kkk in range(len(bcDict_t[iB])):
# print kkk
# #print bcDict_t[iB][kkk][1]
# #print bcDict_t[iB][kkk][1][1]
# print bcDict_t[iB].values()
# subBCDict.update({bcDict_t[iB][kkk][0]:bcDict_t[iB][kkk][1]})
# bcDict[iB] = subBCDict.copy()
# #bcDict[iB] = subBCDict.deepcopy()
#print bcDict
#sys.exit(0)
return bcDict_t
def _delim_list(self, sep):
quoted = quotedString.copy()
value = quoted | Word(printables, excludeChars=('%' + sep))
parser = delimitedList(value, delim=sep)
return parser
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
FTR: this is hideous.
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(
replaceWith('1')) | Keyword('no', caseless=True).setParseAction(
replaceWith('0'))
def _handle_ip(*x):
a, b, c = x[2]
return ' %s = { %s }' % (a, c[0])
def _handle_diraddr(*x):
a, b, c = x[2]
self._set(DIRADDRESSES, ' %s' % '\n '.join(c))
return
def np(words, fn=gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_name = np((NAME, ), action=lambda x: self._set_name(x[2]))
gr_address = np((ADDRESS, ), action=self._parse_setter(ADDRESS))
gr_fd_conn = np(PList('fd connect timeout'), gr_number,
self._parse_setter(FD_CONNECT_TIMEOUT, True))
gr_heart = np(PList('heartbeat interval'), gr_number,
self._parse_setter(HEARTBEATINTERVAL, True))
gr_max_con = np(PList('maximum console connections'), gr_number,
self._parse_setter(MAXIMUMCONSOLECONNECTIONS, True))
gr_max_jobs = np(PList('maximum concurrent jobs'),
gr_number,
action=self._parse_setter(MAXIMUMCONCURRENTJOBS,
True))
gr_pass = np((PASSWORD, ), action=self._parse_setter(PASSWORD))
gr_pid = np(PList('pid directory'),
action=self._parse_setter(PIDDIRECTORY))
gr_query = np(PList('query file'),
action=self._parse_setter(QUERYFILE))
gr_scripts = np(PList('scripts directory'),
action=self._parse_setter(SCRIPTS_DIRECTORY))
gr_sd_conn = np(PList('sd connect timeout'), gr_number,
self._parse_setter(SD_CONNECT_TIMEOUT, True))
gr_source = np(PList('source address'),
action=self._parse_setter(SOURCEADDRESS))
gr_stats = np(PList('statistics retention'),
action=self._parse_setter(STATISTICS_RETENTION))
gr_verid = np((VERID, ), action=self._parse_setter(VERID))
gr_messages = np(
(MESSAGES, ),
action=lambda x: self._parse_setter(MESSAGE_ID, dereference=True))
gr_work_dir = np(PList('working directory'),
action=self._parse_setter(WORKINGDIRECTORY))
gr_port = np(PList('dir port'), gr_number,
self._parse_setter(PORT, True))
gr_monitor = np((MONITOR, ), gr_yn, action=self._parse_setter(MONITOR))
# This is a complicated one
da_addr = np(('Addr', 'Port'), Word(printables),
lambda x, y, z: ' '.join(z))
da_ip = np(
('IPv4', 'IPv6', 'IP'),
nestedExpr(
'{', '}',
OneOrMore(da_addr).setParseAction(
lambda x, y, z: ' ; '.join(z)))).setParseAction(_handle_ip)
da_addresses = np(PList('dir addresses'),
nestedExpr('{', '}', OneOrMore(da_ip)),
_handle_diraddr)
gr_res = OneOrMore(gr_name | gr_address | gr_fd_conn | gr_heart
| gr_max_con | gr_max_jobs | gr_pass | gr_pid
| gr_query | gr_scripts | gr_sd_conn | gr_source
| gr_stats | gr_verid | gr_messages | gr_work_dir
| gr_port | gr_monitor | da_addresses)
result = gr_res.parseString(string, parseAll=True)
return 'Director: ' + self[NAME]
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(
replaceWith('1')) | Keyword('no', caseless=True).setParseAction(
replaceWith('0'))
def np(words, fn=gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME, ), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np(PList('pool type'),
action=self._parse_setter(POOLTYPE))
gr_line = gr_line | np(PList('maximum volumes'),
action=self._parse_setter(MAXIMUMVOLUMES))
gr_line = gr_line | np((STORAGE, ), action=self._parse_setter(STORAGE))
gr_line = gr_line | np(PList('use volume once'),
gr_yn,
action=self._parse_setter(USEVOLUMEONCE))
gr_line = gr_line | np(PList('catalog files'),
gr_yn,
action=self._parse_setter(CATALOGFILES))
gr_line = gr_line | np(
PList('auto prune'), gr_yn, action=self._parse_setter(AUTOPRUNE))
gr_line = gr_line | np(
(RECYCLE, ), gr_yn, action=self._parse_setter(RECYCLE))
gr_line = gr_line | np(PList('recycle oldest volume'),
gr_yn,
action=self._parse_setter(RECYCLEOLDESTVOLUME))
gr_line = gr_line | np(PList('recycle current volume'),
gr_yn,
action=self._parse_setter(RECYCLECURRENTVOLUME))
gr_line = gr_line | np(PList('purge oldest volume'),
gr_yn,
action=self._parse_setter(PURGEOLDESTVOLUME))
gr_line = gr_line | np(PList('maximum volume jobs'),
gr_number,
action=self._parse_setter(MAXIMUMVOLUMEJOBS))
gr_line = gr_line | np(PList('maximum volume files'),
gr_number,
action=self._parse_setter(MAXIMUMVOLUMEFILES))
gr_line = gr_line | np(PList('maximum volume bytes'),
action=self._parse_setter(MAXIMUMVOLUMEBYTES))
gr_line = gr_line | np(PList('volume use duration'),
action=self._parse_setter(VOLUMEUSEDURATION))
gr_line = gr_line | np(PList('volume retention'),
action=self._parse_setter(VOLUMERETENTION))
gr_line = gr_line | np(PList('action on purge'),
action=self._parse_setter(ACTIONONPURGE))
gr_line = gr_line | np(PList('scratch pool'),
action=self._parse_setter(SCRATCHPOOL))
gr_line = gr_line | np(PList('recycle pool'),
action=self._parse_setter(RECYCLEPOOL))
gr_line = gr_line | np(PList('file retention'),
action=self._parse_setter(FILERETENTION))
gr_line = gr_line | np(PList('job retention'),
action=self._parse_setter(JOBRETENTION))
gr_line = gr_line | np(PList('cleaning prefix'),
action=self._parse_setter(CLEANINGPREFIX))
gr_line = gr_line | np(PList('label format'),
action=self._parse_setter(LABELFORMAT))
gr_res = OneOrMore(gr_line)
result = gr_res.parseString(string, parseAll=True)
return 'Pool: ' + self[NAME]
def parser():
eq = Keyword("==").suppress()
ne = Keyword("!=").suppress()
gte = Keyword(">=").suppress()
lte = Keyword("<=").suppress()
gt = Keyword(">").suppress()
lt = Keyword("<").suppress()
lparen = Keyword("(").suppress()
rparen = Keyword(")").suppress()
dotdot = Keyword("..").suppress()
colon = Keyword(":").suppress()
assignOp = Keyword('=').suppress()
sign = Word("+-", exact=1)
nonzero = ''.join([str(i) for i in range(1, 10)])
True_ = Word("true").setParseAction(lambda *args: True)
False_ = Word("false").setParseAction(lambda *args: False)
Boolean_ = True_ | False_
Int_ = Combine(Optional(sign) + Word(nonzero, nums)).setParseAction(lambda l, s, t: int(t[0]))
Hex_ = Regex("0x[A-Fa-f0-9]+").setResultsName('hex')
Term_ = Word(alphas, alphanums + "_")
VarToken_ = Combine('$' + Term_)
Var_ = Group(VarToken_).setResultsName('var')
String_ = quotedString.copy().setParseAction(removeQuotes).setResultsName('string')
Val_ = Forward()
Vals_ = Group(delimitedList(Val_, delim=' ')).setResultsName('vals')
In_ = Group(Val_ + Keyword("in", caseless=True).suppress() + lparen + Vals_ + rparen).setResultsName('in')
Eq_ = Group(Val_ + eq + Val_).setResultsName('eq')
Ne_ = Group(Val_ + ne + Val_).setResultsName('ne')
Gte_ = Group(Val_ + gte + Val_).setResultsName('gte')
Lte_ = Group(Val_ + lte + Val_).setResultsName('lte')
Gt_ = Group(Val_ + gt + Val_).setResultsName('gt')
Lt_ = Group(Val_ + lt + Val_).setResultsName('lt')
Range_ = Group(lparen +
Optional(Val_, default=None).setResultsName('min') + dotdot +
Optional(Val_, default=None).setResultsName('max') +
rparen).setResultsName('range')
Or_ = Forward()
Parens_ = Group(Suppress("(") + Or_ + Suppress(")")).setResultsName("parens") | Val_
Not_ = Forward()
Not_ << (Group(Suppress(Keyword("not", caseless=True)) + Not_
).setResultsName("not") | Parens_)
And_ = Forward()
And_ << (Group(Not_ + Suppress(Keyword("and", caseless=True)) + And_
).setResultsName("and") | Not_)
Or_ << (Group(And_ + Suppress(Keyword("or", caseless=True)) + Or_
).setResultsName("or") | And_)
Assign_ = Group(VarToken_ + assignOp + Val_).setResultsName('assign')
Contract_ = Regex(
'([A-Za-z_][A-Za-z0-9_]*)'
'\(((?:0x[A-Fa-f0-9]+)'
'|(?:\$[A-Za-z_][A-Za-z0-9_]*))\)').setResultsName('contract')
Send_ = Group((Keyword("send") | Keyword("asend"))
+ Optional(Int_) + (Contract_ | Var_) + Optional(Var_ | Term_)
+ Optional(Vals_)).setResultsName('send')
Call_ = Group(Keyword("call")
+ Optional(Int_) + (Var_ | Contract_) + Optional(Var_ | Term_)
+ Optional(Vals_)).setResultsName('call')
Address_ = Group(Keyword("address")
+ (Var_ | Contract_)).setResultsName('address')
Deploy_ = Group(Keyword("deploy") + (Var_ | Term_) + (Var_ | Term_)).setResultsName('deploy')
Expression_ = (And_ | Or_ | Eq_ | Ne_ | Gte_
| Lte_ | Gt_ | Lt_ | In_ | Range_
| Call_ | Deploy_ | Send_)
Val_ << (Hex_ | Int_ | Boolean_ | String_ | Var_ | Expression_)
Statement_ = (Contract_ | Assign_ | Send_ | Call_ | Deploy_ | Val_)
return Statement_
def parse_string(self, string):
'''Populate a new object from a string.
Parsing is hard, so we're going to call out to the pyparsing
library here. I hope you installed it!
'''
from pyparsing import quotedString, restOfLine, Keyword, nestedExpr, OneOrMore, Word, Literal, removeQuotes, nums, replaceWith, printables
gr_eq = Literal('=')
gr_stripped_string = quotedString.copy().setParseAction(removeQuotes)
gr_opt_quoted_string = gr_stripped_string | restOfLine
gr_number = Word(nums)
gr_yn = Keyword('yes', caseless=True).setParseAction(
replaceWith('1')) | Keyword('no', caseless=True).setParseAction(
replaceWith('0'))
def _handle_ip(*x):
a, b, c = x[2]
return ' %s = { %s }' % (a, c[0])
def _handle_fdaddr(*x):
a, b, c = x[2]
self._set(FDADDRESSES, ' %s' % '\n '.join(c))
return
def np(words, fn=gr_opt_quoted_string, action=None):
p = Keyword(words[0], caseless=True)
for w in words[1:]:
p = p | Keyword(w, caseless=True)
p = p + gr_eq + fn
p.setParseAction(action)
return p
gr_line = np((NAME, ), action=lambda x: self._set_name(x[2]))
gr_line = gr_line | np((ADDRESS, ), action=self._parse_setter(ADDRESS))
gr_line = gr_line | np(
(CATALOG, ),
action=self._parse_setter(CATALOG_ID, dereference=True))
gr_line = gr_line | np(
(PASSWORD, ), action=self._parse_setter(PASSWORD))
gr_line = gr_line | np(PList('file retention'),
action=self._parse_setter(FILERETENTION))
gr_line = gr_line | np(PList('job retention'),
action=self._parse_setter(JOBRETENTION))
gr_line = gr_line | np(
(PRIORITY, ), gr_number, action=self._parse_setter(PRIORITY))
gr_line = gr_line | np(PList('working directory'),
action=self._parse_setter(WORKINGDIRECTORY))
gr_line = gr_line | np(PList('pid directory'),
action=self._parse_setter(PIDDIRECTORY))
gr_line = gr_line | np(PList('heart beat interval'),
action=self._parse_setter(HEARTBEATINTERVAL))
gr_line = gr_line | np(PList('fd address'),
action=self._parse_setter(FDADDRESS))
gr_line = gr_line | np(PList('fd source address'),
action=self._parse_setter(FDSOURCEADDRESS))
gr_line = gr_line | np(PList('pki key pair'),
action=self._parse_setter(PKIKEYPAIR))
gr_line = gr_line | np(PList('pki master key'),
action=self._parse_setter(PKIMASTERKEY))
gr_line = gr_line | np(
PList('fd port'), gr_number, action=self._parse_setter(FDPORT))
gr_line = gr_line | np(
PList('auto prune'), gr_yn, action=self._parse_setter(AUTOPRUNE))
gr_line = gr_line | np(PList('maximum concurrent jobs'),
gr_number,
action=self._parse_setter(FDPORT))
gr_line = gr_line | np(PList('pki encryption'),
gr_yn,
action=self._parse_setter(PKIENCRYPTION))
gr_line = gr_line | np(PList('pki signatures'),
gr_yn,
action=self._parse_setter(PKISIGNATURES))
# This is a complicated one
da_addr = np(('Addr', 'Port'), Word(printables),
lambda x, y, z: ' '.join(z))
da_ip = np(
('IPv4', 'IPv6', 'IP'),
nestedExpr(
'{', '}',
OneOrMore(da_addr).setParseAction(
lambda x, y, z: ' ; '.join(z)))).setParseAction(_handle_ip)
da_addresses = np(('fd addresses', FDADDRESSES),
nestedExpr('{', '}', OneOrMore(da_ip)),
_handle_fdaddr)
gr_res = OneOrMore(gr_line | da_addresses)
result = gr_res.parseString(string, parseAll=True)
return 'Client: ' + self[NAME]
