def create_method(self, line):
indent = self.tree.find_indent(line) #this should always be zero
if line[:4] == 'def ': # this prevents us from overlooking consecutively declared methods
self.tree.indent = indent
element, attributes = parse_definition(line[4:])
if element[0] not in string.letters + '_' or not (element.replace('_','').isalnum()):
raise ParserError("Method name must be alphanumeric with underscores and start with a letter or underscore")
if element in self.valid_tags.keys():
raise ParserError("Can't create method named '%s', it's a reserved markup element name" % element)
self.creating_method = element
# methods access shadowed variables to prevent overwriting globals
self.method_map[element] = Method(attributes, True, self.color, element)
else: # regular line inside a method
if indent == 0:
self.creating_method = None
return True #finished
else:
# check if this is a loop:
if not self.current_verbatim and (self.loop_stack or line.strip()[:4] == 'for '):
self.create_loop(line[len(self.tree.indent_marker):])
else:
if self.need_to_remove_method_vars:
self.need_to_remove_method_vars = False
line = '%s%s(%s)' % (self.tree.indent_to(indent), self.current_verbatim, ', '.join(self.verbatim_vars[GLOBAL_VARS]))
# add normal line to method sequence
self.method_map[self.creating_method].add_line(line[len(self.tree.indent_marker):],
self.current_verbatim,
self.verbatim_vars[METHOD_VARS])
return False
def parse_template_engine_call(self, line, indent):
compressed = line.replace(' ','')
try:
# invoke template engine method
if indent is None:
whitespace = ''
else:
whitespace = self.tree.indent_to(indent)
method_pair = line.split('(')[0].rstrip().rsplit('.', 1)
end_method = self.template_engines[method_pair[0]].end_method(method_pair[1])
if indent is not None:
if end_method:
# full-line declaration
ending_tag = whitespace + end_method
else:
# invocation that's a subset of the line
ending_tag = None
self.handle_indent(indent, ending_tag)
if not self.template_engines[method_pair[0]].is_submethod(method_pair[1], indent):
# this is a method created using 'create()'
#end_method = self.template_engines[method_pair[0]].end_method(method_pair[1])
if end_method is None:
if indent is not None:
self.tree.indent = indent
else:
if indent is None:
# this will happen when we try something like:
# div(src=django.for(a, b))
raise ParserError("'%s' method requires a closing tag, it can't be used as part of other logic on the line" % method_pair[1])
#self.handle_indent(indent, whitespace + end_method)
else:
if indent is None:
raise ParserError("Can't use '%s' as part of other logic on the line because it's a branch of an existing method" % method_pair[1])
# this is a sub-method of a verbatim method (i.e. 'else' is a submethod of 'if')
# submethods are created using 'append()'
#note the indent+1, we want to close inner logic, but not the method itself
endtag = self.template_engines[method_pair[0]].end_method(method_pair[1], False)
if endtag is None:
self.handle_indent(indent+1, None)
self.element_stack.pop()
else:
self.handle_indent(indent+1, whitespace + endtag)
attr = get_attr(line)
return whitespace + self.template_engines[method_pair[0]].call_method(method_pair[1], attr, indent)
#self.element_stack.append(whitespace + self.template_engines[method_pair[0]].end_method(method_pair[1]))
#except (KeyError, IndexError, TypeError):
except ParserError:
raise ParserError("Improper TemplateEngine method declaration or invocation")
# repeated from the definition method, rewrite to make the code more DRY
# helper method for checking if the line involves variable assignment and/or reassignment
if line.find(':=') != -1:
# variable declaration
self.set_variable(line)
return
elif line.find('+=') != -1 or line.find('-=') != -1 or line.find('*=') != -1 or line.find('/=') != -1:
# variable increment
res = self.expand_assignment_ops(line)
line = self.var_map[res]
def create_template_engine(self, line):
# creates a new set of rules for a templating engine, such as Django, Web2py, or Rails
pair = line.split('=')
if len(pair) != 2:
raise ParserError("Improper TemplateEngine declaration")
elif pair[0].isalnum() and pair[0] not in string.letters:
raise ParserError("TemplateEngine must have alphanumeric name that starts with a letter")
template = pair[1][pair[1].find('(')+1:pair[1].rfind(')')-1].strip().strip("'").strip('"')
self.template_engines[pair[0].rstrip()] = TemplateEngine(template)
def set_variable(self, tag):
# sets the variable(s) in the system
vars = tag.split(':=')
if len(vars) == 1:
raise ParserError("You're trying to declare variable %s without assignment" % vars[0].strip())
text = vars[-1].lstrip()
for var in vars[:-1]:
if var[0] == '$':
self.var_map[var.rstrip()] = self.get_variables(text)
else:
raise ParserError("Illegal assignment to a constant '%s'" % var.rstrip())
def create_loop(self, line):
# we can just piggy-back on create_method, since a loop is essentially a repeated function
# the only tricky part is that loops can be nested
tag = line.strip()
indent = self.tree.find_indent(line)
if tag[:4] == 'for ': # new loop started (either within old loop, or outside)
self.handle_indent(indent, None)
var = tag.split()[1] #[for,$var,in,...]
if var in self.var_map.keys():
raise ParserError("Can't reuse previously defined variable %s as loop iterator" % var)
#array = self.get_variables(tag.split('[')[1].split(']')[0]).split(',')
#iterator = re.findall('^for[ ]+(\$[a-zA-Z_][a-zA-Z0-9_]*)[ ]+in', tag)[0]
#array = self.get_variables(tag, [var]).split('[', 1)[1].rsplit(']', 1)[0].split(',')
array = get_attr('(%s)' % self.get_variables(tag, [var]).split('[', 1)[1].rsplit(']', 1)[0])
for i in range(len(array)):
array[i] = array[i].strip()
loop_name = 'rapydml_loop_def_%s' % self.loop_index
self.loop_stack.append((loop_name, indent, array))
self.method_map[loop_name] = Method([var], False, self.color, loop_name) # loops see/access global var space
self.loop_index += 1
else: # command inside the loop or outside (loop termination)
loop_name = self.loop_stack[-1][0]
loop_indent = self.loop_stack[-1][1]
indent_diff = indent-loop_indent
if indent_diff < 1:
# loop terminated, unroll it and execute this line as normal
self.unroll_loop()
self.handle_line(line)
else:
self.tree.indent = indent
self.method_map[loop_name].add_line(line[len(self.tree.indent_to(loop_indent+1)):])
def close_last_element(self):
# closes last html tag
tag = self.element_stack.pop()
if tag is None:
return # this is not an element that requires closing
# in order to use short-hand <tag /> we need to make sure that tag is not a special tag,
# and that the name matches as well as indent
if self.last_opened_element is None:
tag_type = -1
else:
try:
tag_type = self.valid_tags[self.last_opened_element][0]
except KeyError:
try:
tag_type = self.valid_tags['*'][0]
except KeyError:
# WE SHOULD NOT GET IN HERE UNLESS SOMETHING IS WRONG
print "This logic should not trigger, please inform RapydML developers, provide the contents of your .pyml file as well."
raise ParserError("'%s' is not a valid markup tag or method name." % self.last_opened_element)
if tag_type == NORMAL \
and re.search('^%s</%s>' % (self.tree.indent_to(self.tree.indent), self.last_opened_element), tag):
self.write(' />\n', -2)
elif tag_type != SINGLE or self.last_opened_element is None:
self.write(tag)
def parse_template_engine_method_declaration(line):
command_pair = line.split('=', 1)
if len(command_pair) != 2:
raise ParserError("Improper TemplateEngine method declaration, variable assigned to must follow format of TemplateEngine.MethodName")
method_pair = command_pair[0].rstrip().rsplit('.', 1)
method, attributes = parse_definition(command_pair[1])
return method_pair[0], method_pair[1], method, attributes
def expand_assignment_ops(self, tag, perform=True):
# takes operation of form '$a += 3', converts it to '$a := $a + 3' and evaluates it, assigning new value to $a
result, tag = expand_assignment(tag)
pair = tag.split(':=')
if len(pair) > 2:
raise ParserError("Command '%s' has multiple assignment operators, invalid syntax" % tag)
self.var_map[result] = do_arithmetic(self.get_variables(pair[1].strip()))
return result
def resolve_indexes(self, line):
# replace all indexes with corresponding values
while line.count(']['):
try:
parts = line.split('][')
array, start_index = parse_array_part(parts[0]+']')
index = parts[1].split(']')[0] #this can trigger IndexError
num = int(index) #this can trigger ValueError
line = line.replace(line[start_index:len('%s[%s]' % (parts[0],index))+1], array[num])
except ValueError:
# this occurs if we try something like array['abcd'] or array[3.5]
raise ParserError("Invalid index '%s', index must be an integer" % index)
except IndexError:
# this occurs if we try some improperly terminated line like array[ or ][
raise ParserError("Syntax error while trying to parse index")
return line
def call_method(self, method, vars, indent):
# this returns an actual template tag to use for this method call
if len(vars) != self.methods[method][1]:
raise ParserError("TemplateEngine method %s takes %d variables, %d given" % (method, self.methods[method][1], len(vars)))
self.handle_indent(indent, method)
vars = tuple(vars)
contents = self.methods[method][0] % vars
return self.tag_format % contents
def do_arithmetic(operation):
# this function solves simple arithmetic such as +/-/*//
# if we detect # or non-numeric variable, that's not a method, try to run it through color converter
# valid color formats we expect are '#ffffff', '#000', or 'white' (note the quotes, we assume them necessary)
# what about style?
try:
result = eval(operation)
return repr(result)
except SyntaxError, NameError:
raise ParserError("Command '%s' is not a valid mathematical operation" % operation.strip())
def handle_verbatim_declaration(self, tag):
assignment_pair = tag.split('=', 1)
newtag = assignment_pair[0].strip()
element, attributes = parse_definition(assignment_pair[1])
length = len(attributes)
# code_block follows same format as verbatim, but always takes additional argument
if element == 'code_block':
length -= 1
sys_command = attributes.pop()[1:-1]
if length == 0:
# no args were passed, this version has no outer tags wrapping the text
starttag = endtag = ''
elif length == 1:
# received 1 argument, it's a tag to use as a template
# TODO: eventually we want this to also check declared methods, if available, first
#if attributes[0] in self.template_engines.keys():
# endtag = self.template_engines[attributes[0]].tag_format %
#else:
# endtag = '</%s>\n' % attributes[0]
tagname, tagattr = parse_definition(attributes[0])
starttag, endtag = create_tag(tagname, tagattr)
elif length == 2:
# received 2 quoted arguments for beginning and end tags
starttag = attributes[0][1:-1] + '\n'
endtag = attributes[1][1:-1] + '\n'
else:
if element == 'code_block':
raise ParserError("Code Block definition takes 1, 2, or 3 arguments, %s arguments were given" % length)
else:
raise ParserError("Verbatim definition takes 0, 1, or 2 arguments, %s arguments were given" % length)
# append to verbatim format in (start_tag, end_tag, type) format
if element == 'verbatim_line':
self.verbatim[newtag] = (starttag, endtag, SINGLE_LINE)
elif element == 'verbatim':
self.verbatim[newtag] = (starttag, endtag, MULTI_LINE)
else:
self.verbatim[newtag] = (starttag, endtag, CODE_BLOCK, sys_command)
def import_module(self, line):
tokens = line.split()
if len(tokens) != 2 or tokens[0] != 'import':
raise ParserError("Invalid import statement: %s" % line.strip())
if tokens[1] not in self.imported_files:
try:
self.imported_files.append(tokens[1])
self.parse(tokens[1].replace('.', '/') +'.pyml', True)
except IOError:
# failed importing from working directory, try importing from rapydml directory
cur_dir = os.getcwd()
try:
# we have to rely on __file__, because cwd could be different if invoked by another script
os.chdir(os.path.dirname(__file__))
self.imported_files.append(tokens[1])
self.parse(tokens[1].replace('.', '/') +'.pyml', True)
except IOError:
raise ParserError("Can't import '%s', module doesn't exist" % tokens[1])
finally:
os.chdir(cur_dir)
def to_num(self, color):
# first we standardize colors to 6-digit hex format
color = color.lower()
if color in self.color_map.keys():
color = self.color_map[color]
elif len(color) == 3:
color = '%s%s%s%s%s%s' % (color[0], color[0], color[1], color[1], color[2], color[2])
elif len(color) != 6:
raise ParserError("Color '%s' is not a valid HTML color" % color)
# now we return base 10 representation of the number
return int(color, 16)
def parse_template_engine_definition(self, line):
compressed = line.replace(' ','')
if compressed.find('create(') != -1:
# add a template to existing template engine
engine, template, method_call, attr = parse_template_engine_method_declaration(line)
try:
if len(attr) == 1:
self.template_engines[engine].add_method(template, attr[0][1:-1])
else:
self.template_engines[engine].add_method(template, attr[0][1:-1], attr[1][1:-1])
except KeyError:
raise ParserError("Attempting to add a method to a TemplateEngine prior to declaration")
else: # append()
# append additional logic to a template
engine, template, method_call, attr = parse_template_engine_method_declaration(line)
original_pair = method_call.split('.') # we only care about 1st 2 args
if len(original_pair) != 3:
raise ParserError("Method being appended to must follow TemplateEngine.MethodName format")
try:
self.template_engines[original_pair[0]].enhance_method(original_pair[1], template)
except KeyError:
raise ParserError("Attempting to append functionality to a non-existing method")
try:
self.template_engines[engine].add_method(template, attr[0][1:-1])
except KeyError:
raise ParserError("Attempting to add a method to a TemplateEngine prior to declaration")
# helper method for checking if the line involves variable assignment and/or reassignment
if line.find(':=') != -1:
# variable declaration
self.set_variable(line)
return
elif line.find('+=') != -1 or line.find('-=') != -1 or line.find('*=') != -1 or line.find('/=') != -1:
# variable increment
res = self.expand_assignment_ops(line)
line = self.var_map[res]
def handle_indent(self, indent, method_name):
# push to or pop from the stack, depending on indent
# first method call
if self.method_stack:
indent_diff = indent-self.method_stack[-1][1]
else:
self.method_stack.append((method_name, indent))
return
if self.is_submethod(method_name, indent): #indent_diff == 0 and method_name in self.methods[self.method_stack[-1][0]][3]:
# this is a submethod of current method
#self.method_stack.append((self.method_stack[-1][0], indent))
pass
elif indent_diff < 1: #pop
while indent_diff < 1:
self.method_stack.pop()
indent_diff += 1
self.method_stack.append((method_name, indent))
elif indent_diff > 1:
raise ParserError('Incorrect indentation')
def replace_variables(code, var_hash, ignore_list=[]):
#plugs the variables into the line
vars = re.findall('(?<!\\\\)\$[A-Za-z_][A-Za-z0-9_]*', code)
for var in ignore_list:
try:
vars.remove(var)
except ValueError:
pass # variable doesn't appear on this line
for var in vars:
try:
# the first version will not replace 2nd occurence in strings like '$a$a'
#code = re.sub('(?<!(\\\\|[A-Za-z0-9_]))\%s(?![A-Za-z0-9_])' % var, var_hash[var], code)
code = re.sub('(?<!\\\\)\%s(?![A-Za-z0-9_])' % var, var_hash[var], code)
except KeyError:
raise ParserError("Variable '%s' used prior to definition" % var)
if code.find('python.') != -1:
# use of python method
code = eval_python(code)
return code
def run_method(self, args, heap):
if self.copy_heap:
self.heap = heap.copy()
else:
self.heap = heap
#var_hash = {} #if we 'clutter' the global heap, it makes some logic easier and allows more Python-like reuse of variables after loop terminates
for i in range(len(self.attributes)):
#TEMP: the \ replacing is a temporary quickfix for misunderstood problem of Python interpreting the string when it shouldn't
#var_hash[self.attributes[i]] = args[i].replace('\\','\\\\')
try:
self.heap[self.attributes[i]] = args[i].replace('\\','\\\\')
except IndexError:
raise ParserError("Method '%s' expects %s attributes, %s given." % \
(self.name, len(self.attributes), len(args)))
for line in self.lines:
if line[0] == VERBATIM:
verbatim_line = line[1]
# replace variables we specified
for variable in line[2]:
verbatim_line = re.sub(r'\%s(?![A-Za-z0-9_])' % variable, self.heap[variable], verbatim_line)
yield verbatim_line # don't run any logic on verbatim lines
else:
line = line[1]
# this is where we handle replacing method arguments
assignments = line.count(':=')
if assignments == 1:
operands = line.split(':=')
self.heap[operands[0].strip()] = self.eval_line(operands[1].strip())
yield None # this line produces no output
elif assignments > 1:
raise "Multiple assignments on same line aren't allowed"
else:
yield self.eval_line(line)
def handle_line(self, line):
indent = self.tree.find_indent(line)
if DEBUG:
print self.element_stack, '|%s|' % self.creating_method, indent, repr(line)
whitespace = self.tree.indent_to(indent)
#parse the tag
tag = line.strip()
#if tag[0] == '$' and tag.find(':=') == -1:
# tag = self.get_variables(tag)
if self.current_verbatim is not None or \
tag.split('(')[0].strip(':') in self.verbatim.keys():
# verbatim call
if self.loop_stack and tag.split('(')[0].strip(':') in self.verbatim.keys():
# in case verbatim ended a loop, we want to handle that
# we can return right after, since the loop terminator will automatically
# call handle_line() again with verbatim tag
self.create_loop(line)
return
else:
# note: even comments inside verbatim block get treated verbatim
last_line_processed = self.handle_verbatim_call(line)
# if we're inside method creation, we want to keep going, so this line gets added to
# the method
if not self.creating_method or last_line_processed:
return
elif not tag or tag[0] == '#':
# strip comments and blank lines
return
line = expand_arrays(line)
tag = line.strip()
# first check is a quick pre-qualifier to avoid expensive regex, second one avoids
# false positives like: this_is_not_verbatim_call()
if (tag.find('verbatim') != -1 or tag.find('code_block') != -1) \
and re.search(r'\b(verbatim(_line)?|code_block)\b', tag):
# verbatim/code_block declaration
self.handle_verbatim_declaration(tag)
return
elif self.creating_method is None and (self.loop_stack or tag[:4] == 'for '):
# loop
self.create_loop(line)
return
elif self.creating_method is not None or line[:4] == 'def ':
# method definition
finished = self.create_method(line)
if not finished:
return
if tag.find(':=') != -1:
# variable declaration
self.set_variable(tag)
return
elif line.find('+=') != -1 or line.find('-=') != -1 or line.find('*=') != -1 or line.find('/=') != -1:
# variable increment
res = self.expand_assignment_ops(line)
tag = self.var_map[res]
elif line[:7] == 'import ':
# import
self.import_module(line)
return
elif line.find('TemplateEngine') != -1:
# template class declaration
self.create_template_engine(tag)
return
elif tag.find('.') != -1:
if tag.find('python.') != -1:
# use of python method
line = eval_python(line)
tag = line.strip()
# continue parsing, there could be other instances of '.'
if tag.find('.append(') != -1 or tag.find('create(') != -1:
# template engine declaration
self.parse_template_engine_definition(tag)
return
else:
# this regex will find all occurences of template engine calls in the form of:
# template_engine.template_method(.*)
# as long as the method is not in a string
# and matching the parentheses correctly even if items inside use parentheses, up to 1 level deep
substitutions = re.findall(r'(\b[A-Za-z_][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*\b%s)' % REGEX_NESTED_PAREN, tag)
if len(substitutions) == 1 and tag.find(substitutions[0][1]) == 0:
line = self.parse_template_engine_call(tag, indent)
else:
for item in substitutions:
pattern = item.replace('(', '\(').replace(')', '\)').replace('.', '\.')
if re.match(r'%s\b%s' % (NOT_SINGLE_QUOTED, pattern), tag) \
and item.split('.')[0] in self.template_engines:
# template method declaration or call
tag = tag.replace(item, self.parse_template_engine_call(item, None).strip('\n'))
# if substitutions:
# self.write(line)
# return
elif line == EOF_MARKER:
return
tag = self.get_variables(tag)
if tag[0] in ('"', "'"):
# handle quoted strings as plain-text
starttag = tag[1:-1] + '\n'
element = None
htmlend = None
else:
# test if this tag is a method call, if so execute it
element, attributes = parse_definition(tag)
if element in self.method_map.keys():
self.handle_indent(indent, None)
for method_line in self.method_map[element].run_method(attributes, self.var_map):
if method_line is not None:
self.handle_line(whitespace+method_line)
return
else:
# this is a regular tag, not a method, let's make sure the element and attributes are valid
try:
self.valid_tags[element]
hash_key = element
except KeyError:
try:
self.valid_tags['*'] # if we can't access this, wildcard element was not declared
hash_key = '*'
except KeyError:
raise ParserError("'%s' is not a valid markup tag or method name." % element)
if self.valid_tags[hash_key][1] is not None:
for attr in attributes:
attr_name = attr.split('=', 1)[0]
if attr_name not in self.valid_tags[hash_key][1]:
raise ParserError("'%s' is not one of allowed attributes for '%s' element" % (attr_name, hash_key))
starttag, endtag = create_tag(element, attributes)
htmlend = whitespace + endtag
# check indent difference, close old tags if indent < 1
self.handle_indent(indent, htmlend)
# update variables
self.last_opened_element = element
# dump the current line to file
self.write(whitespace + starttag)
def handle_verbatim_call(self, line):
indent = self.tree.find_indent(line)
if self.current_verbatim is None:
# this is the first line of verbatim logic
# set verbatim, get replaceable variables and check that they exist
self.current_verbatim, verbatim_vars = parse_definition(line.strip())
self.verbatim_vars = ([], [])
for variable in verbatim_vars:
try:
self.var_map[variable]
self.verbatim_vars[GLOBAL_VARS].append(variable)
except KeyError:
if self.creating_method \
and (variable in self.method_map[self.creating_method].attributes \
or variable in self.method_map[self.creating_method].local_vars):
self.verbatim_vars[METHOD_VARS].append(variable)
self.need_to_remove_method_vars = True
continue
raise ParserError("Variable '%s' used prior to declaration." % variable)
self.verbatim_indent = self.tree.find_indent(line)
if not self.creating_method:
self.handle_indent(indent, None)
self.last_opened_element = None
self.element_stack.append(None)
else:
# we're continuing to parse existing verbatim logic
verbatim_properties = self.verbatim[self.current_verbatim]
if indent > self.verbatim_indent:
# still inside verbatim block
if not self.creating_method:
line = line[self.verbatim_indent+1:]
# plug in the variables, if they appear on this line
for variable in self.verbatim_vars[GLOBAL_VARS]:
line = re.sub(r'\%s(?![A-Za-z0-9_])' % variable, self.var_map[variable], line)
self.verbatim_buffer += line
else:
# end of verbatim logic
if not self.creating_method:
if verbatim_properties[2] == SINGLE_LINE:
self.verbatim_buffer = re.sub('\n[ ]*', ' ', self.verbatim_buffer)
self.verbatim_buffer += '\n'
elif verbatim_properties[2] == CODE_BLOCK:
result = Popen('echo %s' % pipes.quote(self.verbatim_buffer) + verbatim_properties[3],
stdout=PIPE, stderr=PIPE, shell=True).communicate()
if result[1]:
raise ShellError("'%s' code_block tag triggered the following OS error: %s" %
(self.current_verbatim, result[1]))
self.verbatim_buffer = result[0] + '\n'
if verbatim_properties[1] != '':
vindent = self.tree.indent_to(self.verbatim_indent)
self.verbatim_buffer = '%s%s%s%s%s' % (\
vindent,
verbatim_properties[0],
self.verbatim_buffer,
vindent,
verbatim_properties[1])
self.write(self.verbatim_buffer)
self.verbatim_buffer = ''
self.close_last_element() # close verbatim element so it does not screw up the stack
self.current_verbatim = None
self.handle_line(line)
return True
return False