def articulate_re(text, articulate, articulators, depth=0): "Articulate text using a simple regular expression lexer" bottom = len(articulators) # move down the stack until a pattern is matched ... while True: texts = [t for t in articulators[depth].split(text) if t] depth += 1 if len(texts) > 1: break if depth == bottom: # no match found, bottom of the stack reached. return text field = set() if depth == bottom: # bottom of the stack reached, simply split name = pns_model.pns_name(netstring.encode(texts), field) else: # not yet at the bottom of the stack, recurse ... name = pns_model.pns_name( netstring.encode((articulate_re(t, articulate, articulators, depth) for t in texts)), field ) # validate the articulated name(s) as a Public Names if len(field) > 1: articulate((len(field), field, name, text)) return name
def articulate_chunk(text, articulate, articulators, CHUNK, depth=0): "Articulate in chunks, usefull for larger text articulations" bottom = len(articulators) # move down the stack until a pattern is matched ... while True: texts = [t for t in articulators[depth].split(text) if t] depth += 1 if len(texts) > 1: break if depth == bottom: # no match found, bottom of the stack reached, this # is supposed to chunk, not articulate, do not name # because it is inarticulated junk ... return "" if depth == bottom: # bottom of the stack reached, simply split do not articulate # any further, there is nothing to chunk and it is a flat # articulation ... return pns_model.pns_name(netstring.encode((t for t in texts if len(t) <= CHUNK)), set()) # not yet at the bottom of the stack, recurse for t in texts: if len(t) > CHUNK: articulate_chunk(t, articulate, articulators, CHUNK, depth) else: articulate_re(t, articulate, articulators, depth) return None
def xml_utf8_name (element, dom): # articulate a context's name one from its children's name(s) field = set () return pns_model.pns_name (netstring.encode (( child.pns_name for child in element.xml_children if child.pns_name )), field)
from allegra import netstring, pns_model context = '7:Rushing,3:Sam,' print context == pns_model.pns_name (context, set ()) context = '3:Sam,7:Rushing,' print context == pns_model.pns_name (context, set ()) context = '4:Kill,4:Kill,8:Pussycat,' print context == pns_model.pns_name (context, set ()) field = set () context = netstring.netstrings (( 'An', 'example', 'of', ( 'Simple', 'Articulated', 'Text' ), 'and', 'a', 'bit', 'of', 'dispersion' )) print netstring.netoutline (context) name = pns_model.pns_name (context, field) print netstring.netoutline (name) print 'field: %r' % field print 'horizon: %d' % len (field)
def xml_utf8_name(element, dom): # articulate a context's name one from its children's name(s) field = set() return pns_model.pns_name( netstring.encode((child.pns_name for child in element.xml_children if child.pns_name)), field)
def pns_articulate_route (encoded): names = netstring.netlist (encoded) name = pns_model.pns_name (netstring.encode (names[1:]), set ()) return (names[0], name)