def test05get_identifier(self): ''' Test get_identifier. ''' self.assertEqual(get_identifier(''), ('', 0)) self.assertEqual(get_identifier('a'), ('a', 1)) self.assertEqual(get_identifier('a1'), ('a1', 2)) self.assertEqual(get_identifier('1a'), ('', 0)) self.assertEqual(get_identifier('1a', 1), ('a', 2))
def parse_option(opttext): ''' Parse an option string into an option name and value. ''' option, offset = get_identifier(opttext) if len(option) == 0: raise ValueError('missing option name') option = option.lower() opttext = opttext[offset:] if not opttext: raise ValueError("missing option value") if opttext.startswith('='): opttext = aptarg[1:] elif opttext[0].isspace(): opttext = opttext.strip() else: raise ValueError("invalid text after option: %r", opttext) return option, opttext
def parse_inner(T, s, offset, stopchar, prefix): ''' Parse hashname:hashhextext from `s` at offset `offset`. Return HashCode instance and new offset. ''' hashname, offset = get_identifier(s, offset) if not hashname: raise ValueError("missing hashname at offset %d" % (offset, )) hashclass = HASHCLASS_BY_NAME[hashname] if offset >= len(s) or s[offset] != ':': raise ValueError("missing colon at offset %d" % (offset, )) offset += 1 hexlen = hashclass.HASHLEN * 2 hashtext = s[offset:offset + hexlen] if len(hashtext) != hexlen: raise ValueError("expected %d hex digits, found only %d" % (hexlen, len(hashtext))) offset += hexlen H = hashclass.from_hashbytes_hex(hashtext) return H, offset
def parse(self, s, offset=0): ''' Parse an object from the string `s` starting at `offset`. Return the object and the new offset. Parameters: * `s`: the source string * `offset`: optional string offset, default 0 ''' # strings value, offset2 = self.parse_qs(s, offset, optional=True) if value is not None: return value, offset2 # decimal values if s[offset:offset + 1].isdigit(): return get_decimal_or_float_value(s, offset) # {json} if s.startswith('{', offset): sub = s[offset:] m, suboffset = pfx_call(json.JSONDecoder().raw_decode, sub) offset += suboffset return m, offset # prefix{....} prefix, offset = get_identifier(s, offset) if not prefix: raise ValueError("no type prefix at offset %d" % (offset, )) with Pfx("prefix %r", prefix): if offset >= len(s) or s[offset] != '{': raise ValueError("missing opening '{' at offset %d" % (offset, )) offset += 1 baseclass = self.prefix_map.get(prefix) if baseclass is None: raise ValueError("prefix not registered: %r" % (prefix, )) with Pfx("baseclass=%s", baseclass.__name__): o, offset = baseclass.parse_inner(self, s, offset, '}', prefix) if offset > len(s): raise ValueError("parse_inner returns offset beyond text") if offset >= len(s) or s[offset] != '}': raise ValueError("missing closing '}' at offset %d" % (offset, )) offset += 1 return o, offset
def parseMacro(context, text=None, offset=0): ''' Parse macro from `text` from `FileContext` `context` at `offset`. Return `(MacroTerm,offset)`. ''' if text is None: text = context.text mmark = None mtext = None param_mexprs = [] modifiers = [] mpermute = False mliteral = False try: if text[offset] != '$': raise ParseError(context, offset, 'expected "$" at start of macro') offset += 1 ch = text[offset] # $x if ch == '_' or ch.isalpha( ) or ch in SPECIAL_MACROS or ch in TARGET_MACROS: offset += 1 M = MacroTerm(context, ch), offset return M # $(foo) or ${foo} if ch == '(': mmark = ch mmark2 = ')' elif ch == '{': mmark = ch mmark2 = '}' else: raise ParseError(context, offset, 'invalid special macro "%s"', ch) # $((foo)) or ${{foo}} ? offset += 1 ch = text[offset] if ch == mmark: mpermute = True offset += 1 _, offset = get_white(text, offset) mtext, offset = get_identifier(text, offset) if mtext: # $(macro_name) # check for macro parameters _, offset = get_white(text, offset) if text[offset] == '(': # $(macro_name(param,...)) offset += 1 _, offset = get_white(text, offset) while text[offset] != ')': mexpr, offset = MacroExpression.parse(context, text=text, offset=offset, stopchars=',)') param_mexprs.append(mexpr) _, offset = get_white(text, offset) if text[offset] == ',': # gather comma and following whitespace _, offset = get_white(text, offset + 1) continue if text[offset] != ')': raise ParseError( context, offset, 'macro parameters: expected comma or closing parenthesis, found: %s', text[offset:]) offset += 1 else: # must be "qtext" or a special macro name q = text[offset] if q == '"' or q == "'": # $('qstr') mliteral = True offset += 1 text_offset = offset while text[offset] != q: offset += 1 mtext = text[text_offset:offset] offset += 1 elif q in SPECIAL_MACROS or q in TARGET_MACROS: # $(@ ...) etc mtext = q offset += 1 else: raise ParseError(context, offset, 'unknown special macro name "%s"', q) _, offset = get_white(text, offset) # collect modifiers while True: try: ch = text[offset] if ch == mmark2: # macro closing bracket break if ch.isspace(): # whitespace offset += 1 continue if ch == '?': raise ParseError( context, offset, 'bare query "?" found in modifiers at: %s', text[offset:]) mod0 = ch modargs = () with Pfx(mod0): offset0 = offset offset += 1 if mod0 == 'D': modclass = ModDirpart elif mod0 == 'E': modclass = ModEval elif mod0 == 'F': modclass = ModFilepart elif mod0 == 'G': modclass = ModGlob if offset < len(text) and text[offset] == '?': offset += 1 modargs = ( False, True, ) else: modargs = ( False, False, ) elif mod0 == 'g': modclass = ModGlob if offset < len(text) and text[offset] == '?': offset += 1 modargs = ( True, True, ) else: modargs = ( True, False, ) elif mod0 in 'PpSs': if offset < len(text) and text[offset] == '[': offset += 1 if offset >= len(text): raise ParseError(context, offset, 'missing separator') sep = text[offset] offset += 1 if offset >= len(text) or text[offset] != ']': raise ParseError(context, offset, 'missing closing "]"') offset += 1 else: sep = '.' modargs = (sep, ) if mod0 == 'P': modclass = ModPrefixLong elif mod0 == 'p': modclass = ModPrefixShort elif mod0 == 'S': modclass = ModSuffixShort elif mod0 == 's': modclass = ModSuffixLong else: raise NotImplementedError( "parse error: unhandled PpSs letter \"%s\"" % (mod0, )) elif mod0 == '<': modclass = ModFromFiles if offset < len(text) and text[offset] == '?': offset += 1 modargs = (True, ) else: modargs = (False, ) elif mod0 in '-+*': modclass = ModSetOp _, offset = get_white(text, offset) q = text[offset:offset + 1] if q == '"' or q == "'": # 'qstr' offset += 1 text_offset = offset while text[offset] != q: offset += 1 mtext = text[text_offset:offset] offset += 1 modargs = (mod0, mtext, True) else: submname, offset = get_identifier(text, offset) if not submname: raise ParseError( context, offset, 'missing macro name or string after "%s" modifier', mod0) modargs = (mod0, submname, False) elif mod0 == ':': _, offset = get_white(text, offset) if offset >= len(text): raise ParseError( context, offset, 'missing opening delimiter in :,ptn,rep,') delim = text[offset] if delim == mmark2: raise ParseError( context, offset, 'found closing bracket instead of leading delimiter in :,ptn,rep,' ) if delim.isalnum(): raise ParseError( context, offset, 'invalid delimiter in :,ptn,rep, - must be nonalphanumeric' ) modclass = ModSubstitute offset += 1 try: ptn, repl, etc = text[offset:].split(delim, 2) except ValueError: raise ParseError(context, offset, 'incomplete :%sptn%srep%s', delim, delim, delim) offset = len(text) - len(etc) modargs = (ptn, repl) else: invert = False if ch == '!': invert = True # !/regexp/ or !{commalist}? _, offset2 = get_white(text, offset) if offset2 == len( text) or text[offset2] not in '/{': raise ParseError( context, offset2, '"!" not followed by /regexp/ or {comma-list} at %r', text[offset2:]) offset = offset2 ch = text[offset] if ch == '/': modclass = ModSelectRegexp offset += 1 mexpr, end = MacroExpression.parse(context, text=text, offset=offset, stopchars='/') if end >= len(text): raise ParseError(context, offset, 'incomplete /regexp/: %r', text[offset:]) assert text[end] == '/' offset = end + 1 modargs = (mexpr, invert) else: raise ParseError( context, offset0, 'unknown macro modifier "%s": "%s"', mod0, text[offset0:]) modifiers.append( modclass(context, text[offset0:offset], *modargs)) except ParseError as e: error("%s", e) offset += 1 assert ch == mmark2, "should be at \"%s\", but am at: %s" % ( mmark, text[offset:]) offset += 1 if mpermute: if offset >= len(text) or text[offset] != mmark2: raise ParseError(context, offset, 'incomplete macro closing brackets') else: offset += 1 M = MacroTerm(context, mtext, modifiers, param_mexprs, permute=mpermute, literal=mliteral) return M, offset except IndexError: raise ParseError(context, offset, 'parse incomplete, offset=%d, remainder: %s', offset, text[offset:]) raise ParseError(context, offset, 'unhandled parse failure at offset %d: %s', offset, text[offset:])
def readMakefileLines(M, fp, parent_context=None, start_lineno=1, missing_ok=False): ''' Read a Mykefile and yield text lines. This generator parses slosh extensions and :if/ifdef/ifndef/else/endif directives. ''' if isinstance(fp, str): # open file, yield contents filename = fp try: with Pfx("open %r", filename).partial(open, filename)() as fp: for O in readMakefileLines(M, fp, parent_context, missing_ok=missing_ok): yield O except OSError as e: if e.errno == errno.ENOENT or e.errno == errno.EPERM: yield parent_context, e return try: filename = fp.name except AttributeError: filename = str(fp) ifStack = [] # active ifStates (state, in-first-branch) context = None # FileContext(filename, lineno, line) prevline = None for lineno, line in enumerate(fp, start_lineno): if not line.endswith('\n'): raise ParseError(context, len(line), '%s:%d: unexpected EOF (missing final newline)', filename, lineno) if prevline is not None: # prepend previous continuation line if any # keep the same FileContext line = prevline + '\n' + line prevline = None else: # start of line - new FileContext context = FileContext(filename, lineno, line.rstrip(), parent_context) with Pfx(str(context)): if line.endswith('\\\n'): # continuation line - gather next line before parse prevline = line[:-2] continue # skip blank lines and comments w1 = line.lstrip() if not w1 or w1.startswith('#'): continue try: # look for :if etc if line.startswith(':'): # top level directive _, offset = get_white(line, 1) word, offset = get_identifier(line, offset) if not word: raise SyntaxError("missing directive name") _, offset = get_white(line, offset) with Pfx(word): if word == 'ifdef': mname, offset = get_identifier(line, offset) if not mname: raise ParseError(context, offset, "missing macro name") _, offset = get_white(line, offset) if offset < len(line): raise ParseError( context, offset, "extra arguments after macro name: %s", line[offset:]) newIfState = [False, True] if all([item[0] for item in ifStack]): newIfState[0] = nsget(M.namespaces, mname) is not None ifStack.append(newIfState) continue if word == "ifndef": mname, offset = get_identifier(line, offset) if not mname: raise ParseError(context, offset, "missing macro name") _, offset = get_white(line, offset) if offset < len(line): raise ParseError( context, offset, "extra arguments after macro name: %s", line[offset:]) newIfState = [True, True] if all([item[0] for item in ifStack]): newIfState[0] = nsget(M.namespaces, mname) is None ifStack.append(newIfState) continue if word == "if": raise ParseError(context, offset, "\":if\" not yet implemented") continue if word == "else": # extra text permitted if not ifStack: raise ParseError( context, 0, ":else: no active :if directives in this file" ) if not ifStack[-1][1]: raise ParseError(context, 0, ":else inside :else") ifStack[-1][1] = False continue if word == "endif": # extra text permitted if not ifStack: raise ParseError( context, 0, ":endif: no active :if directives in this file" ) ifStack.pop() continue if word == "include": if all(ifState[0] for ifState in ifStack): if offset == len(line): raise ParseError( context, offset, ":include: no include files specified") include_mexpr = MacroExpression.from_text( context, offset=offset) for include_file in include_mexpr( context, M.namespaces).split(): if len(include_file) == 0: continue if isabs(include_file): include_file = os.path.join( dirname(filename), include_file) yield from readMakefileLines( M, include_file, parent_context=context, missing_ok=missing_ok) continue if not all(ifState[0] for ifState in ifStack): # in false branch of "if"; skip line continue except SyntaxError as e: error(e) continue # NB: yield is outside the Pfx context manager because Pfx does # not play nicely with generators yield context, line if prevline is not None: # incomplete continuation line error("%s: unexpected EOF: unterminated slosh continued line") if ifStack: raise SyntaxError("%s: EOF with open :if directives" % (filename, ))
def get_store_spec(s, offset=0): ''' Get a single Store specification from a string. Return `(matched, type, params, offset)` being the matched text, store type, parameters and the new offset. Recognised specifications: * `"text"`: Quoted store spec, needed to enclose some of the following syntaxes if they do not consume the whole string. * `[clause_name]`: The name of a clause to be obtained from a Config. * `/path/to/something`, `./path/to/something`: A filesystem path to a local resource. Supported paths: - `.../foo.sock`: A UNIX socket based StreamStore. - `.../dir`: A DataDirStore directory. - `.../foo.vtd `: (STILL TODO): A VTDStore. * `|command`: A subprocess implementing the streaming protocol. * `store_type(param=value,...)`: A general Store specification. * `store_type:params...`: An inline Store specification. Supported inline types: `tcp:[host]:port` TODO: * `ssh://host/[store-designator-as-above]`: * `unix:/path/to/socket`: Connect to a daemon implementing the streaming protocol. * `http[s]://host/prefix`: A Store presenting content under prefix: + `/h/hashcode.hashtype`: Block data by hashcode + `/i/hashcode.hashtype`: Indirect block by hashcode. * `s3://bucketname/prefix/hashcode.hashtype`: An AWS S3 bucket with raw blocks. ''' offset0 = offset if offset >= len(s): raise ValueError("empty string") if s.startswith('"', offset): # "store_spec" qs, offset = get_qstr(s, offset, q='"') _, store_type, params, offset2 = get_store_spec(qs, 0) if offset2 < len(qs): raise ValueError("unparsed text inside quotes: %r" % (qs[offset2:], )) elif s.startswith('[', offset): # [clause_name] store_type = 'config' clause_name, offset = get_ini_clausename(s, offset) params = {'clause_name': clause_name} elif s.startswith('/', offset) or s.startswith('./', offset): path = s[offset:] offset = len(s) if path.endswith('.sock'): store_type = 'socket' params = {'socket_path': path} elif isdirpath(path): store_type = 'datadir' params = {'path': path} elif isfilepath(path): store_type = 'datafile' params = {'path': path} else: raise ValueError("%r: not a directory or a socket" % (path, )) elif s.startswith('|', offset): # |shell command store_type = 'shell' params = {'shcmd': s[offset + 1:].strip()} offset = len(s) else: store_type, offset = get_identifier(s, offset) if not store_type: raise ValueError("expected identifier at offset %d, found: %r" % (offset, s[offset:])) with Pfx(store_type): if s.startswith('(', offset): params, offset = get_params(s, offset) elif s.startswith(':', offset): offset += 1 params = {} if store_type == 'tcp': colon2 = s.find(':', offset) if colon2 < offset: raise ValueError( "missing second colon after offset %d" % (offset, )) hostpart = s[offset:colon2] offset = colon2 + 1 if not isinstance(hostpart, str): raise ValueError( "expected hostpart to be a string, got: %r" % (hostpart, )) if not hostpart: hostpart = 'localhost' params['host'] = hostpart portpart, offset = get_token(s, offset) params['port'] = portpart else: raise ValueError("unrecognised Store type for inline form") else: raise ValueError("no parameters") return s[offset0:offset], store_type, params, offset
def parse_mapping(self, s, offset=0, stopchar=None, required=None, optional=None): ''' Parse a mapping from the string `s`. Return the mapping and the new offset. Parameters: * `s`: the source string * `offset`: optional string offset, default 0 * `stopchar`: ending character, not to be consumed * `required`: if specified, validate that the mapping contains all the keys in this list * `optional`: if specified, validate that the mapping contains no keys which are not required or optional If `required` or `optional` is specified the return takes the form: offset, required_values..., optional_values... where missing optional values are presented as None. ''' if optional is not None and required is None: raise ValueError("required is None but optional is specified: %r" % (optional, )) d = OrderedDict() while offset < len(s) and (stopchar is None or s[offset] != stopchar): k, offset = get_identifier(s, offset) if not k: raise ValueError("offset %d: not an identifier" % (offset, )) if offset >= len(s) or s[offset] != ':': raise ValueError("offset %d: expected ':'" % (offset, )) offset += 1 v, offset = self.parse(s, offset) d[k] = v if offset >= len(s): break c = s[offset] if c == stopchar: break if c != ',': raise ValueError("offset %d: expected ',' but found: %r" % (offset, s[offset:])) offset += 1 if required is None and optional is None: return d, offset for k in required: if k not in d: raise ValueError("missing required field %r" % (k, )) if optional is not None: for k in d.keys(): if k not in required and k not in optional: raise ValueError("unexpected field %r" % (k, )) ret = [offset] for k in required: ret.append(d[k]) for k in optional: ret.append(d.get(k)) return ret
def parse(self, fp, parent_context=None, missing_ok=False): ''' Read a Mykefile and yield Macros and Targets. ''' from .make import Target, Action action_list = None # not in a target for context, line in readMakefileLines(self, fp, parent_context=parent_context, missing_ok=missing_ok): with Pfx(str(context)): if isinstance(line, OSError): e = line if e.errno == errno.ENOENT or e.errno == errno.EPERM: if missing_ok: continue e.context = context yield e break raise e try: if line.startswith(':'): # top level directive _, doffset = get_white(line, 1) word, offset = get_identifier(line, doffset) if not word: raise ParseError(context, doffset, "missing directive name") _, offset = get_white(line, offset) with Pfx(word): if word == 'append': if offset == len(line): raise ParseError(context, offset, "nothing to append") mexpr, offset = MacroExpression.parse( context, line, offset) assert offset == len(line) for include_file in mexpr( context, self.namespaces).split(): if include_file: if not os.path.isabs(include_file): include_file = os.path.join( realpath(dirname(fp.name)), include_file) self.add_appendfile(include_file) continue if word == 'import': if offset == len(line): raise ParseError(context, offset, "nothing to import") ok = True missing_envvars = [] for envvar in line[offset:].split(): if envvar: envvalue = os.environ.get(envvar) if envvalue is None: error("no $%s" % (envvar, )) ok = False missing_envvars.append(envvar) else: yield Macro( context, envvar, (), envvalue.replace('$', '$$')) if not ok: raise ValueError( "missing environment variables: %s" % (missing_envvars, )) continue if word == 'precious': if offset == len(line): raise ParseError( context, offset, "nothing to mark as precious") mexpr, offset = MacroExpression.parse( context, line, offset) self.precious.update(word for word in mexpr( context, self.namespaces).split() if word) continue raise ParseError(context, doffset, "unrecognised directive") if action_list is not None: # currently collating a Target if not line[0].isspace(): # new target or unindented assignment etc - fall through # action_list is already attached to targets, # so simply reset it to None to keep state action_list = None else: # action line _, offset = get_white(line) if offset >= len(line) or line[offset] != ':': # ordinary shell action action_silent = False if offset < len(line) and line[offset] == '@': action_silent = True offset += 1 A = Action(context, 'shell', line[offset:], silent=action_silent) self.debug_parse("add action: %s", A) action_list.append(A) continue # in-target directive like ":make" _, offset = get_white(line, offset + 1) directive, offset = get_identifier(line, offset) if not directive: raise ParseError( context, offset, "missing in-target directive after leading colon" ) A = Action(context, directive, line[offset:].lstrip()) self.debug_parse("add action: %s", A) action_list.append(A) continue try: macro = Macro.from_assignment(context, line) except ValueError: pass else: yield macro continue # presumably a target definition # gather up the target as a macro expression target_mexpr, offset = MacroExpression.parse(context, stopchars=':') if not context.text.startswith(':', offset): raise ParseError(context, offset, "no colon in target definition") prereqs_mexpr, offset = MacroExpression.parse( context, offset=offset + 1, stopchars=':') if offset < len( context.text) and context.text[offset] == ':': postprereqs_mexpr, offset = MacroExpression.parse( context, offset=offset + 1) else: postprereqs_mexpr = [] action_list = [] for target in target_mexpr(context, self.namespaces).split(): yield Target(self, target, context, prereqs=prereqs_mexpr, postprereqs=postprereqs_mexpr, actions=action_list) continue raise ParseError(context, 0, 'unparsed line') except ParseError as e: exception("%s", e) self.debug_parse("finish parse")
def edit_groupness(MDB, addresses, subgroups): ''' Modify the group memberships of the supplied addresses and groups. Removed addresses or groups are not modified. ''' with Pfx("edit_groupness()"): Gs = sorted(set(subgroups), key=lambda G: G.name) As = sorted(set(addresses), key=lambda A: A.realname.lower()) with tempfile.NamedTemporaryFile(suffix='.txt') as T: with Pfx(T.name): with codecs.open(T.name, "w", encoding="utf-8") as ofp: # present groups first for G in Gs: supergroups = sorted(set(G.GROUPs), key=lambda g: g.name) line = u'%-15s @%s\n' % (",".join(supergroups), G.name) ofp.write(line) # present addresses next for A in As: groups = sorted(set(A.GROUPs)) af = A.formatted ab = A.abbreviation if ab: af = "=%s %s" % (ab, af) line = u"%-15s %s\n" % (",".join(groups), af) ofp.write(line) editor = os.environ.get('EDITOR', 'vi') xit = os.system("%s %s" % (editor, cs.sh.quotestr(T.name))) if xit != 0: # TODO: catch SIGINT etc? raise RuntimeError("error editing \"%s\"" % (T.name,)) new_groups = {} with codecs.open(T.name, "r", "utf-8") as ifp: lineno = 0 for line in ifp: lineno += 1 with Pfx("%d", lineno): if not line.endswith("\n"): raise ValueError("truncated file, missing trailing newline") line = line.rstrip() groups, addrtext = line.split(None, 1) groups = [group for group in groups.split(',') if group] if addrtext.startswith('@'): # presume single group name groupname, offset = get_identifier(addrtext, 1) if offset < len(addrtext): warning("invalid @groupname: %r", addrtext) else: MDB.make(('GROUP', groupname)).GROUPs = groups continue # otherwise, address list on RHS As = set() with Pfx(addrtext): for realname, addr in getaddresses((addrtext,)): with Pfx("realname=%r, addr=%r", realname, addr): A = MDB.getAddressNode(addr) if realname.startswith('=' ) and not realname.startswith('=?'): with Pfx(repr(realname)): ab, realname = realname.split(None, 1) ab = ab[1:] if not ab: ab = None else: ab = None try: A.abbreviation = ab except ValueError as e: error(e) # add named groups to those associated with this address new_groups.setdefault(A, set()).update(groups) realname = ustr(realname.strip()) if realname and realname != A.realname: A.realname = realname # apply groups of whichever addresses survived for A, groups in new_groups.items(): if set(A.GROUPs) != groups: # reset .GROUP list if changed A.GROUPs = groups