def main(argv): # This becomes osh-lexer_def.re2c.c. It is compiled to osh-lexer_def.c and then # included. action = argv[1] if action == 'c': # Print code to stdout. TranslateOshLexer(lexer_def.LEXER_DEF) TranslateSimpleLexer('MatchEchoToken', lexer_def.ECHO_E_DEF) TranslateSimpleLexer('MatchGlobToken', lexer_def.GLOB_DEF) TranslateSimpleLexer('MatchPS1Token', lexer_def.PS1_DEF) TranslateSimpleLexer('MatchHistoryToken', lexer_def.HISTORY_DEF) TranslateSimpleLexer('MatchBraceRangeToken', lexer_def.BRACE_RANGE_DEF) #TranslateSimpleLexer('MatchQsnToken', lexer_def.QSN_DEF) # e.g. "pipefail" -> option-I::pipefail pairs = [(opt.name, opt.index) for opt in option_def.All()] StringToInt('MatchOption', pairs) TranslateRegexToPredicate(lexer_def.VAR_NAME_RE, 'IsValidVarName') TranslateRegexToPredicate(lexer_def.SHOULD_HIJACK_RE, 'ShouldHijack') elif action == 'print-all': # Top level is a switch statement. for state, pat_list in lexer_def.LEXER_DEF.iteritems(): print(state) # This level is re2c patterns. for is_regex, pat, token_id in pat_list: print('\t%r -> %r' % (pat, token_id)) if is_regex: #print re_tree out_pat = TranslateRegex(pat) #print out_pat print() elif action == 'print-regex': unique = set() num_regexes = 0 for state, pat_list in lexer_def.LEXER_DEF.iteritems(): print(state) # This level is re2c patterns. for is_regex, pat, token_id in pat_list: #print '\t%r -> %r' % (pat, token_id) if is_regex: print('\t' + pat) print('\t' + TranslateRegex(pat)) print() #PrintRegex(pat) num_regexes += 1 unique.add(pat) else: print('\t' + TranslateConstant(pat)) print() print('Printed %d regexes (%d unique)' % (num_regexes, len(unique)))
def AddOptionsToArgSpec(spec): """Shared between 'set' builtin and the shell's own arg parser.""" for opt in option_def.All(): if opt.builtin == 'set': spec.Option(opt.short_flag, opt.name) elif opt.builtin == 'shopt': # unimplemented options are accepted in bin/osh and in shopt -s foo spec.ShoptOption(opt.name) else: # 'interactive' Has a cell for internal use, but isn't allowed to be # modified. pass # Add strict:all, etc. for name in option_def.META_OPTIONS: spec.ShoptOption(name)
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') # TODO: # generate builtin::echo, etc. # # And in Python do the same. option = _CreateSum(_OPT_ENUM, [opt.name for opt in option_def.All()]) builtin = _CreateSum(_BUILTIN_ENUM, [b.enum_name for b in builtin_def.All()]) # TODO: could shrink array later. # [opt.name for opt in option_def.All() if opt.implemented]) schema_ast = asdl_.Module('option', [], [option, builtin]) if action == 'cpp': from asdl import gen_cpp out_prefix = argv[2] with open(out_prefix + '.h', 'w') as f: f.write("""\ #ifndef OPTION_ASDL_H #define OPTION_ASDL_H namespace option_asdl { """) # TODO: Could suppress option_str v = gen_cpp.ClassDefVisitor(f, {}, simple_int_sums=_SIMPLE) v.VisitModule(schema_ast) f.write(""" } // namespace option_asdl #endif // OPTION_ASDL_H """) with open(out_prefix + '.cc', 'w') as f: f.write("""\ #include <assert.h> #include "option_asdl.h" namespace option_asdl { """) v = gen_cpp.MethodDefVisitor(f, {}, simple_int_sums=_SIMPLE) v.VisitModule(schema_ast) f.write('} // namespace option_asdl\n') elif action == 'mypy': from asdl import gen_python f = sys.stdout f.write("""\ from asdl import pybase """) # option_i type v = gen_python.GenMyPyVisitor(f, None, simple_int_sums=_SIMPLE) v.VisitModule(schema_ast) else: raise RuntimeError('Invalid action %r' % action)
def Build(self, argv, arg, base_opts): """Given flags to complete/compgen, return a UserSpec.""" cmd_ev = self.cmd_ev actions = [] # NOTE: bash doesn't actually check the name until completion time, but # obviously it's better to check here. if arg.F: func_name = arg.F func = cmd_ev.procs.get(func_name) if func is None: raise error.Usage('Function %r not found' % func_name) actions.append( completion.ShellFuncAction(cmd_ev, func, self.comp_lookup)) # NOTE: We need completion for -A action itself!!! bash seems to have it. for name in arg.actions: if name == 'alias': a = _FixedWordsAction(self.parse_ctx.aliases) elif name == 'binding': # TODO: Where do we get this from? a = _FixedWordsAction(['vi-delete']) elif name == 'command': # compgen -A command in bash is SIX things: aliases, builtins, # functions, keywords, external commands relative to the current # directory, and external commands in $PATH. actions.append(_FixedWordsAction(consts.BUILTIN_NAMES)) actions.append(_FixedWordsAction(self.parse_ctx.aliases)) actions.append(_FixedWordsAction(cmd_ev.procs)) actions.append(_FixedWordsAction(lexer_def.OSH_KEYWORD_NAMES)) actions.append(completion.FileSystemAction(exec_only=True)) # Look on the file system. a = completion.ExternalCommandAction(cmd_ev.mem) elif name == 'directory': a = completion.FileSystemAction(dirs_only=True) elif name == 'file': a = completion.FileSystemAction() elif name == 'function': a = _FixedWordsAction(cmd_ev.procs) elif name == 'job': a = _FixedWordsAction(['jobs-not-implemented']) elif name == 'user': a = completion.UsersAction() elif name == 'variable': a = completion.VariablesAction(cmd_ev.mem) elif name == 'helptopic': # Note: it would be nice to have 'helpgroup' for help -i too a = _FixedWordsAction(help_.TOPICS) elif name == 'setopt': names = [ opt.name for opt in option_def.All() if opt.builtin == 'set' ] a = _FixedWordsAction(names) elif name == 'shopt': names = [ opt.name for opt in option_def.All() if opt.builtin == 'shopt' ] a = _FixedWordsAction(names) elif name == 'signal': a = _FixedWordsAction(['TODO:signals']) elif name == 'stopped': a = _FixedWordsAction(['jobs-not-implemented']) else: raise NotImplementedError(name) actions.append(a) # e.g. -W comes after -A directory if arg.W is not None: # could be '' # NOTES: # - Parsing is done at REGISTRATION time, but execution and splitting is # done at COMPLETION time (when the user hits tab). So parse errors # happen early. w_parser = self.parse_ctx.MakeWordParserForPlugin(arg.W) arena = self.parse_ctx.arena try: arg_word = w_parser.ReadForPlugin() except error.Parse as e: ui.PrettyPrintError(e, arena) raise # Let 'complete' or 'compgen' return 2 a = completion.DynamicWordsAction(self.word_ev, self.splitter, arg_word, arena) actions.append(a) extra_actions = [] if base_opts.get('plusdirs'): extra_actions.append(completion.FileSystemAction(dirs_only=True)) # These only happen if there were zero shown. else_actions = [] if base_opts.get('default'): else_actions.append(completion.FileSystemAction()) if base_opts.get('dirnames'): else_actions.append(completion.FileSystemAction(dirs_only=True)) if not actions and not else_actions: raise error.Usage('No actions defined in completion: %s' % argv) p = completion.DefaultPredicate if arg.X: filter_pat = arg.X if filter_pat.startswith('!'): p = completion.GlobPredicate(False, filter_pat[1:]) else: p = completion.GlobPredicate(True, filter_pat) return completion.UserSpec(actions, extra_actions, else_actions, p, prefix=arg.P or '', suffix=arg.S or '')
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') # TODO: Remove duplication in core/meta.py ID_TO_KIND = {} BOOL_ARG_TYPES = {} TEST_UNARY_LOOKUP = {} TEST_BINARY_LOOKUP = {} TEST_OTHER_LOOKUP = {} ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES) id_kind_def.AddKinds(ID_SPEC) id_kind_def.AddBoolKinds(ID_SPEC) # must come second id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP) ids = ID_SPEC.id_str2int.items() ids.sort(key=lambda pair: pair[1]) # Sort by ID if action == 'c': for name, id_int in ids: print('#define id__%s %s' % (name, id_int)) elif action == 'cpp': from asdl import gen_cpp schema_ast = _CreateModule(ID_SPEC, ids) out_prefix = argv[2] with open(out_prefix + '.h', 'w') as f: f.write("""\ #ifndef ID_KIND_ASDL_H #define ID_KIND_ASDL_H namespace id_kind_asdl { """) v = gen_cpp.ClassDefVisitor(f, e_suffix=False, simple_int_sums=['Id']) v.VisitModule(schema_ast) f.write(""" } // namespace id_kind_asdl #endif // ID_KIND_ASDL_H """) with open(out_prefix + '.cc', 'w') as f: f.write("""\ #include <assert.h> #include "id_kind_asdl.h" namespace id_kind_asdl { """) v = gen_cpp.MethodDefVisitor(f, e_suffix=False, simple_int_sums=['Id']) v.VisitModule(schema_ast) f.write('} // namespace id_kind_asdl\n') elif action == 'mypy': from asdl import gen_python schema_ast = _CreateModule(ID_SPEC, ids) #print(schema_ast) f = sys.stdout f.write("""\ from asdl import pybase """) # Minor style issue: we want Id and Kind, not Id_e and Kind_e v = gen_python.GenMyPyVisitor(f, e_suffix=False, simple_int_sums=['Id']) v.VisitModule(schema_ast) elif action == 'cpp-consts': from frontend import consts from _devbuild.gen.id_kind_asdl import Id_str, Kind_str from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str LIST_INT = [ 'STRICT_ALL', 'OIL_BASIC', 'OIL_ALL', 'DEFAULT_TRUE', 'PARSE_OPTION_NUMS', 'SHOPT_OPTION_NUMS', 'SET_OPTION_NUMS', ] # TODO: These could be changed to numbers LIST_STR = ['VISIBLE_SHOPT_NAMES'] prefix = argv[2] with open(prefix + '.h', 'w') as f: def out(fmt, *args): print(fmt % args, file=f) out("""\ #ifndef CONSTS_H #define CONSTS_H #include "mylib.h" #include "id_kind_asdl.h" #include "option_asdl.h" #include "runtime_asdl.h" #include "types_asdl.h" namespace consts { """) for name in LIST_INT: out('extern List<int>* %s;', name) for name in LIST_STR: out('extern List<Str*>* %s;', name) out("""\ extern int NO_INDEX; int RedirDefaultFd(id_kind_asdl::Id_t id); types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id); types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id); id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id); option_asdl::builtin_t LookupNormalBuiltin(Str* s); option_asdl::builtin_t LookupAssignBuiltin(Str* s); option_asdl::builtin_t LookupSpecialBuiltin(Str* s); bool IsControlFlow(Str* s); bool IsKeyword(Str* s); Str* LookupCharC(Str* c); Str* LookupCharPrompt(Str* c); Str* OptionName(option_asdl::option_t opt_num); Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch); } // namespace consts #endif // CONSTS_H """) with open(prefix + '.cc', 'w') as f: def out(fmt, *args): print(fmt % args, file=f) out("""\ #include "consts.h" namespace Id = id_kind_asdl::Id; using id_kind_asdl::Kind; using types_asdl::redir_arg_type_e; using types_asdl::bool_arg_type_e; using option_asdl::builtin_t; namespace consts { int NO_INDEX = 0; // duplicated from frontend/consts.py """) # Note: could use opt_num:: instead of raw ints for name in LIST_INT: val = getattr(consts, name) val_str = ', '.join(str(i) for i in val) out('List<int>* %s = new List<int>({%s});', name, val_str) for name in LIST_STR: val = getattr(consts, name) val_str = '/* TODO */' out('List<Str*>* %s = new List<Str*>({%s});', name, val_str) out("""\ int RedirDefaultFd(id_kind_asdl::Id_t id) { // relies on "switch lowering" switch (id) { """) for id_ in sorted(consts.REDIR_DEFAULT_FD): a = Id_str(id_).replace('.', '::') b = consts.REDIR_DEFAULT_FD[id_] out(' case %s: return %s;' % (a, b)) out("""\ } } """) out("""\ types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) { // relies on "switch lowering" switch (id) { """) for id_ in sorted(consts.REDIR_ARG_TYPES): a = Id_str(id_).replace('.', '::') # redir_arg_type_e::Path, etc. b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace( '.', '_e::') out(' case %s: return %s;' % (a, b)) out("""\ } } """) out("""\ types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) { // relies on "switch lowering" switch (id) { """) for id_ in sorted(BOOL_ARG_TYPES): a = Id_str(id_).replace('.', '::') # bool_arg_type_e::Str, etc. b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::') out(' case %s: return %s;' % (a, b)) out("""\ } } """) out("""\ Kind GetKind(id_kind_asdl::Id_t id) { // relies on "switch lowering" switch (id) { """) for id_ in sorted(ID_TO_KIND): a = Id_str(id_).replace('.', '::') b = Kind_str(ID_TO_KIND[id_]).replace('.', '::') out(' case %s: return %s;' % (a, b)) out("""\ } } """) b = builtin_def.BuiltinDict() GenBuiltinLookup(b, 'LookupNormalBuiltin', 'normal', f) GenBuiltinLookup(b, 'LookupAssignBuiltin', 'assign', f) GenBuiltinLookup(b, 'LookupSpecialBuiltin', 'special', f) # TODO: Fill these in out("""\ bool IsControlFlow(Str* s) { assert(0); } bool IsKeyword(Str* s) { assert(0); } """) GenCharLookup('LookupCharC', consts._ONE_CHAR_C, f, required=True) GenCharLookup('LookupCharPrompt', consts._ONE_CHAR_PROMPT, f) # OptionName() is a bit redundant with ADSL's option_str(), but we can # remove that. out("""\ Str* OptionName(option_asdl::option_t opt_num) { const char* s; switch (opt_num) { """) # These are the only ones we use set_opts = [(opt.index, opt.name) for opt in option_def.All() if opt.builtin == 'set'] for index, name in set_opts: out(' case %s:' % index) out(' s = "%s";' % name) out(' break;') out("""\ default: assert(0); } return new Str(s); // TODO-intern } """) # # Generate a tightly packed 2D array for C, from a Python dict. # edges = consts._IFS_EDGES max_state = max(edge[0] for edge in edges) max_char_kind = max(edge[1] for edge in edges) edge_array = [] for i in xrange(max_state + 1): # unused cells get -1 edge_array.append(['-1'] * (max_char_kind + 1)) for i in xrange(max_state + 1): for j in xrange(max_char_kind + 1): entry = edges.get((i, j)) if entry is not None: # pack (new_state, action) into 32 bits edge_array[i][j] = '(%d<<16)|%d' % entry parts = [] for i in xrange(max_state + 1): parts.append(' {') parts.append(', '.join('%10s' % cell for cell in edge_array[i])) parts.append(' },\n') out("""\ int _IFS_EDGE[%d][%d] = { %s }; """ % (max_state + 1, max_char_kind + 1, ''.join(parts))) out("""\ // Note: all of these are integers, e.g. state_i, emit_i, char_kind_i using runtime_asdl::state_t; using runtime_asdl::emit_t; using runtime_asdl::char_kind_t; Tuple2<state_t, emit_t> IfsEdge(state_t state, runtime_asdl::char_kind_t ch) { int cell = _IFS_EDGE[state][ch]; state_t new_state = cell >> 16; emit_t emit = cell & 0xFFFF; return Tuple2<state_t, emit_t>(new_state, emit); } """) out("""\ } // namespace consts """) elif action == 'py-consts': # It's kind of weird to use the generated code to generate more code. # Can we do this instead with the parsed module for "id" and "types.asdl"? from frontend import consts from _devbuild.gen.id_kind_asdl import Id_str, Kind_str from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str print(""" from _devbuild.gen.id_kind_asdl import Id, Kind from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e """) if 0: print('') print('REDIR_DEFAULT_FD = {') for id_ in sorted(consts.REDIR_DEFAULT_FD): v = consts.REDIR_DEFAULT_FD[id_] print(' %s: %s,' % (Id_str(id_), v)) print('}') print('') print('REDIR_ARG_TYPES = {') for id_ in sorted(consts.REDIR_ARG_TYPES): v = consts.REDIR_ARG_TYPES[id_] # HACK v = redir_arg_type_str(v).replace('.', '_e.') print(' %s: %s,' % (Id_str(id_), v)) print('}') print('') print('BOOL_ARG_TYPES = {') for id_ in sorted(BOOL_ARG_TYPES): v = BOOL_ARG_TYPES[id_] # HACK v = bool_arg_type_str(v).replace('.', '_e.') print(' %s: %s,' % (Id_str(id_), v)) print('}') print('') print('TEST_UNARY_LOOKUP = {') for op_str in sorted(TEST_UNARY_LOOKUP): v = Id_str(TEST_UNARY_LOOKUP[op_str]) print(' %r: %s,' % (op_str, v)) print('}') print('') print('TEST_BINARY_LOOKUP = {') for op_str in sorted(TEST_BINARY_LOOKUP): v = Id_str(TEST_BINARY_LOOKUP[op_str]) print(' %r: %s,' % (op_str, v)) print('}') print('') print('TEST_OTHER_LOOKUP = {') for op_str in sorted(TEST_OTHER_LOOKUP): v = Id_str(TEST_OTHER_LOOKUP[op_str]) print(' %r: %s,' % (op_str, v)) print('}') print('') print('ID_TO_KIND = {') for id_ in sorted(ID_TO_KIND): v = Kind_str(ID_TO_KIND[id_]) print(' %s: %s,' % (Id_str(id_), v)) print('}') else: raise RuntimeError('Invalid action %r' % action)