def _parse_cut_ami(text):
"""Parse "cut ami" command using pyparsing"""
# Word == single token
edctoken = Word(alphanums + '_')
withtoken = Word(printables.replace('=', ''))
preamble = Suppress(Literal('cut') + 'ami')
# e.g. prod-edx-exdapp. Combining into 1 token enforces lack of whitespace
e_d_c = Combine(
edctoken('environment') + '-' + edctoken('deployment') + '-' +
edctoken('cluster'))
# e.g. cut ami for prod-edx-edxapp. Subsequent string literals are converted when added to a pyparsing object.
for_from = Suppress('for') + e_d_c('for_edc') + Suppress(
'from') + e_d_c('from_edc')
# e.g. with foo=bar bing=baz.
# Group puts the k=v pairs in sublists instead of flattening them to the top-level token list.
with_stmt = Suppress('with')
with_stmt += OneOrMore(
Group(withtoken('key') + Suppress('=') +
withtoken('value')))('overrides')
# e.g. using ami-deadbeef
using_stmt = Suppress('using') + Regex('ami-[0-9a-f]{8}')('ami_id')
# 0-1 with and using clauses in any order (see Each())
modifiers = Optional(with_stmt('with_stmt')) & Optional(
using_stmt('using_stmt'))
# 0-1 verbose and noop options in any order (as above)
options = Optional(Literal('verbose')('verbose')) & Optional(
Literal('noop')('noop'))
pattern = StringStart(
) + preamble + options + for_from + modifiers + StringEnd()
parsed = pattern.parseString(text)
return {
'dest_env': parsed.for_edc.environment,
'dest_dep': parsed.for_edc.deployment,
'dest_play': parsed.for_edc.cluster,
'source_env': parsed.from_edc.environment,
'source_dep': parsed.from_edc.deployment,
'source_play': parsed.from_edc.cluster,
'base_ami':
parsed.using_stmt.ami_id if parsed.using_stmt else None,
'version_overrides':
{i.key: i.value
for i in parsed.with_stmt.overrides}
if parsed.with_stmt else None,
'verbose': bool(parsed.verbose),
'noop': bool(parsed.noop),
}
def parse(s):
path = Word(printables) ^ QuotedString('"')
option_name = Literal("exec") ^ Literal("include") ^ Literal("exclude")
option_value = path
option = Group(option_name + option_value)
options_list = Group(ZeroOrMore(option))
path_entry = Group(path + Literal("{") + \
options_list + \
Literal("}"))
path_entries_list = ZeroOrMore(path_entry)
config = StringStart() + path_entries_list + StringEnd()
entries = []
for e in config.parseString(remove_comments(s)):
opts = []
if len(e) == 4:
for o in e[2]:
opts.append(Option(o[0], sanitize(o[1])))
entries.append(Entry(sanitize(e[0]), opts))
return entries
def parse_query_string(self, query_string):
# pylint: disable=too-many-locals
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(f'{alphas}_', f'{alphanums}_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=')
| Literal('<'))
# Value types
value_num = ppc.number
value_bool = (
Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
value_string = QuotedString('"', escQuote='""')
value_orderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then
# combine
# them and convert them to datetime objects
# Date
value_date = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
value_time = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
value_shift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
value_datetime = Combine(
value_date + Optional(value_time) + Optional(value_shift) +
WE(printables.replace('&', ''))
# To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
########################################################################
def validate_time(toks):
"""
Function to convert datetime string into datetime object. The format is
compliant with ParseAction requirements
:param toks: datetime string passed in tokens
:return: datetime object
"""
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dtobj = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError(
'time value has wrong format. The '
'right format is '
'<date>T<time><offset>, '
'where <date> is expressed as '
'[YYYY]-[MM]-[DD], '
'<time> is expressed as [HH]:[MM]:['
'SS], '
'<offset> is expressed as +/-[HH]:['
'MM] '
'given with '
'respect to UTC')
if dtobj.tzinfo is not None and dtobj.utcoffset() is not None:
tzoffset_minutes = int(dtobj.utcoffset().total_seconds() // 60)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(
offset=tzoffset_minutes, name=None)), precision)
return DatetimePrecision(
dtobj.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
value_datetime.setParseAction(validate_time)
# More General types
value = (value_string | value_bool | value_datetime | value_num
| value_orderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it somehow)
value_list = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
single_field = Group(key + operator + value)
list_field = Group(key + (Literal('=in=') | Literal('=notin=')) +
value_list)
orderby_field = Group(key + Literal('=') + value_list)
field = (list_field | orderby_field | single_field)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
general_grammar = SS() + Optional(field) + ZeroOrMore(
separator + field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = general_grammar.parseString(query_string)
# JQuery adds _=timestamp a parameter to not use cached data/response.
# To handle query, remove this "_" parameter from the query string
# For more details check issue #789
# (https://github.com/aiidateam/aiida-core/issues/789) in aiida-core
field_list = [
entry for entry in fields.asList() if entry[0] != '_'
]
except ParseException as err:
raise RestInputValidationError(
'The query string format is invalid. '
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
'number '
'is counted from '
'the first character of the query '
'string.'.format(err))
## return the translator instructions elaborated from the field_list
return self.build_translator_parameters(field_list)
def parse_query_string(query_string):
"""
Function that parse the querystring, extracting infos for limit, offset,
ordering, filters, attribute and extra projections.
:param query_string (as obtained from request.query_string)
:return: parsed values for the querykeys
"""
from pyparsing import Word, alphas, nums, alphanums, printables, \
ZeroOrMore, OneOrMore, Suppress, Optional, Literal, Group, \
QuotedString, Combine, \
StringStart as SS, StringEnd as SE, \
WordStart as WS, WordEnd as WE, \
ParseException
from pyparsing import pyparsing_common as ppc
from dateutil import parser as dtparser
from psycopg2.tz import FixedOffsetTimezone
## Define grammar
# key types
key = Word(alphas + '_', alphanums + '_')
# operators
operator = (Literal('=like=') | Literal('=ilike=') | Literal('=in=')
| Literal('=notin=') | Literal('=') | Literal('!=')
| Literal('>=') | Literal('>') | Literal('<=') | Literal('<'))
# Value types
valueNum = ppc.number
valueBool = (Literal('true')
| Literal('false')).addParseAction(lambda toks: bool(toks[0]))
valueString = QuotedString('"', escQuote='""')
valueOrderby = Combine(Optional(Word('+-', exact=1)) + key)
## DateTimeShift value. First, compose the atomic values and then combine
# them and convert them to datetime objects
# Date
valueDate = Combine(
Word(nums, exact=4) + Literal('-') + Word(nums, exact=2) +
Literal('-') + Word(nums, exact=2))
# Time
valueTime = Combine(
Literal('T') + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)) +
Optional(Literal(':') + Word(nums, exact=2)))
# Shift
valueShift = Combine(
Word('+-', exact=1) + Word(nums, exact=2) +
Optional(Literal(':') + Word(nums, exact=2)))
# Combine atomic values
valueDateTime = Combine(
valueDate + Optional(valueTime) + Optional(valueShift) +
WE(printables.translate(None, '&')) # To us the
# word must end with '&' or end of the string
# Adding WordEnd only here is very important. This makes atomic
# values for date, time and shift not really
# usable alone individually.
)
############################################################################
# Function to convert datetime string into datetime object. The format is
# compliant with ParseAction requirements
def validate_time(s, loc, toks):
datetime_string = toks[0]
# Check the precision
precision = len(datetime_string.replace('T', ':').split(':'))
# Parse
try:
dt = dtparser.parse(datetime_string)
except ValueError:
raise RestInputValidationError("time value has wrong format. The "
"right format is "
"<date>T<time><offset>, "
"where <date> is expressed as "
"[YYYY]-[MM]-[DD], "
"<time> is expressed as [HH]:[MM]:["
"SS], "
"<offset> is expressed as +/-[HH]:["
"MM] "
"given with "
"respect to UTC")
if dt.tzinfo is not None:
tzoffset_minutes = int(
dt.tzinfo.utcoffset(None).total_seconds() / 60)
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=tzoffset_minutes,
name=None)), precision)
else:
return datetime_precision(
dt.replace(tzinfo=FixedOffsetTimezone(offset=0, name=None)),
precision)
########################################################################
# Convert datetime value to datetime object
valueDateTime.setParseAction(validate_time)
# More General types
value = (valueString | valueBool | valueDateTime | valueNum | valueOrderby)
# List of values (I do not check the homogeneity of the types of values,
# query builder will do it in a sense)
valueList = Group(value + OneOrMore(Suppress(',') + value) +
Optional(Suppress(',')))
# Fields
singleField = Group(key + operator + value)
listField = Group(key + (Literal('=in=') | Literal('=notin=')) + valueList)
orderbyField = Group(key + Literal('=') + valueList)
Field = (listField | orderbyField | singleField)
# Fields separator
separator = Suppress(Literal('&'))
# General query string
generalGrammar = SS() + Optional(Field) + ZeroOrMore(separator + Field) + \
Optional(separator) + SE()
## Parse the query string
try:
fields = generalGrammar.parseString(query_string)
field_dict = fields.asDict()
field_list = fields.asList()
except ParseException as e:
raise RestInputValidationError("The query string format is invalid. "
"Parser returned this massage: \"{"
"}.\" Please notice that the column "
"number "
"is counted from "
"the first character of the query "
"string.".format(e))
## return the translator instructions elaborated from the field_list
return build_translator_parameters(field_list)
Sentence).setParseAction(set_head)
Title = (Literal("*") + Sentence).setParseAction(set_title)
Line = StringStart() + (Subhead | Headline | Title)
# Read a todo list, generate a schedule for the scheduler to optimize.
curr_headline = ""
items = []
for line in open(sys.argv[1], "r"):
s.clear()
s["label"] = []
try:
L = Line.parseString(line)
except ParseException, err:
continue
labels = [label.lower() for label in s["label"]]
work = s["content"].replace(" ", "-")
if s["type"] == type_title:
print "_End_Time_: " + "23:59/" + s["content"]
continue
if s["type"] == type_head:
current_headline = s["content"].strip()
# print "Head = ", current_headline
continue
Headline = (Word(nums) + Literal(".") + ZeroOrMore(Label) + Sentence).setParseAction(set_head)
Title = (Literal("*") + Sentence).setParseAction(set_title)
Line = StringStart() + (Subhead | Headline | Title);
# Read a todo list, generate a schedule for the scheduler to optimize.
curr_headline = "";
items = [];
for line in open(sys.argv[1], "r"):
s.clear();
s["label"] = [];
try:
L=Line.parseString(line);
except ParseException,err:
continue;
labels = [label.lower() for label in s["label"]];
work = s["content"].replace(" ", "-");
if s["type"] == type_title:
print "_End_Time_: " + "23:59/" + s["content"];
continue;
if s["type"] == type_head:
current_headline = s["content"].strip();
# print "Head = ", current_headline
continue;
input_file, line_num, oldline, newline))
concat_source.append((input_file, line_num, newline))
#####
# Parse the assembly
#####
logging.info("Parsing source")
asm_addr = 0
assembly = []
labels = {}
label_addrs = {}
for input_file, line_num, line in concat_source:
logging.debug("{:16.16s} {:3d}: IN: {}".format(input_file, line_num,
line))
try:
match = grammar.parseString(line, parseAll=True).asDict()
except ParseException:
print(f"ERROR on line {line_num}: {line}")
raise
logging.debug("{:16.16s} {:3d}: OUT: {}".format(input_file, line_num,
match))
# comments will result in an empty dict
if not match:
continue
# label: the next opcode/data/whatever will be associated with this label
if 'label' in match:
labels[match['label']] = len(assembly)
label_addrs[len(assembly)] = match['label']
logging.debug("{:16.16s} {:3d} Label {} => 0x{:04x}".format(
class Parser:
def __init__( self ):
# define grammar
# definition of a float number
point = Literal('.')
plusOrMinus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine( Optional(plusOrMinus) + number )
floatNumber = Combine( integer + Optional( point + number ) )
# operators
plus = Literal( "+" )
minus = Literal( "-" )
mult = Literal( "*" )
div = Literal( "/" )
# parentheses are discarded
lPar = Literal( "(" ).suppress()
rPar = Literal( ")" ).suppress()
# For precedence handling
addOp = plus | minus
multOp = mult | div
logOp = Literal( "log" )
expr = Forward()
term = Forward()
atom = floatNumber | ( lPar + expr + rPar )
factor = Group(logOp + atom) | atom
term << ( Group(factor + multOp + term ) | factor )
expr << ( Group(term + addOp + expr ) | term )
self.bnf = StringStart() + expr + StringEnd()
self.ops = {
"+" : lambda a, b: a + b,
"-" : lambda a, b: a - b,
"*" : lambda a, b: a * b,
"/" : lambda a, b: a / b
}
self.funcs = {
"log" : lambda a: log10(a)
}
def evaluate( self, node ):
if isinstance(node, basestring):
if node in "+-*/":
return self.ops[node]
elif node in ["log"]:
return self.funcs[node]
else:
return float(node)
else:
if len(node) == 3:
operator = self.evaluate(node[1]);
return operator(self.evaluate(node[0]), self.evaluate(node[2]))
elif len(node) == 2:
function = self.evaluate(node[0]);
return function(self.evaluate(node[1]))
else:
return self.evaluate(node[0])
def parse( self, expr ):
self.exprStack = []
try:
result = self.bnf.parseString( expr )
except ParseException,err:
raise Exception, 'Parse Failure: ' + str(err)
return { "parseResult": result, "evaluation": self.evaluate( result ) }