def main(argv):
# This becomes osh-lexer_def.re2c.c. It is compiled to osh-lexer_def.c and then
# included.
action = argv[1]
if action == 'c':
# Print code to stdout.
TranslateOshLexer(lexer_def.LEXER_DEF)
TranslateSimpleLexer('MatchEchoToken', lexer_def.ECHO_E_DEF)
TranslateSimpleLexer('MatchGlobToken', lexer_def.GLOB_DEF)
TranslateSimpleLexer('MatchPS1Token', lexer_def.PS1_DEF)
TranslateSimpleLexer('MatchHistoryToken', lexer_def.HISTORY_DEF)
TranslateSimpleLexer('MatchBraceRangeToken', lexer_def.BRACE_RANGE_DEF)
#TranslateSimpleLexer('MatchQsnToken', lexer_def.QSN_DEF)
# e.g. "pipefail" -> option-I::pipefail
pairs = [(opt.name, opt.index) for opt in option_def.All()]
StringToInt('MatchOption', pairs)
TranslateRegexToPredicate(lexer_def.VAR_NAME_RE, 'IsValidVarName')
TranslateRegexToPredicate(lexer_def.SHOULD_HIJACK_RE, 'ShouldHijack')
elif action == 'print-all':
# Top level is a switch statement.
for state, pat_list in lexer_def.LEXER_DEF.iteritems():
print(state)
# This level is re2c patterns.
for is_regex, pat, token_id in pat_list:
print('\t%r -> %r' % (pat, token_id))
if is_regex:
#print re_tree
out_pat = TranslateRegex(pat)
#print out_pat
print()
elif action == 'print-regex':
unique = set()
num_regexes = 0
for state, pat_list in lexer_def.LEXER_DEF.iteritems():
print(state)
# This level is re2c patterns.
for is_regex, pat, token_id in pat_list:
#print '\t%r -> %r' % (pat, token_id)
if is_regex:
print('\t' + pat)
print('\t' + TranslateRegex(pat))
print()
#PrintRegex(pat)
num_regexes += 1
unique.add(pat)
else:
print('\t' + TranslateConstant(pat))
print()
print('Printed %d regexes (%d unique)' % (num_regexes, len(unique)))
def AddOptionsToArgSpec(spec):
"""Shared between 'set' builtin and the shell's own arg parser."""
for opt in option_def.All():
if opt.builtin == 'set':
spec.Option(opt.short_flag, opt.name)
elif opt.builtin == 'shopt':
# unimplemented options are accepted in bin/osh and in shopt -s foo
spec.ShoptOption(opt.name)
else:
# 'interactive' Has a cell for internal use, but isn't allowed to be
# modified.
pass
# Add strict:all, etc.
for name in option_def.META_OPTIONS:
spec.ShoptOption(name)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO:
# generate builtin::echo, etc.
#
# And in Python do the same.
option = _CreateSum(_OPT_ENUM, [opt.name for opt in option_def.All()])
builtin = _CreateSum(_BUILTIN_ENUM,
[b.enum_name for b in builtin_def.All()])
# TODO: could shrink array later.
# [opt.name for opt in option_def.All() if opt.implemented])
schema_ast = asdl_.Module('option', [], [option, builtin])
if action == 'cpp':
from asdl import gen_cpp
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef OPTION_ASDL_H
#define OPTION_ASDL_H
namespace option_asdl {
""")
# TODO: Could suppress option_str
v = gen_cpp.ClassDefVisitor(f, {}, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
f.write("""
} // namespace option_asdl
#endif // OPTION_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "option_asdl.h"
namespace option_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {}, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
f.write('} // namespace option_asdl\n')
elif action == 'mypy':
from asdl import gen_python
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# option_i type
v = gen_python.GenMyPyVisitor(f, None, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
else:
raise RuntimeError('Invalid action %r' % action)
def Build(self, argv, arg, base_opts):
"""Given flags to complete/compgen, return a UserSpec."""
cmd_ev = self.cmd_ev
actions = []
# NOTE: bash doesn't actually check the name until completion time, but
# obviously it's better to check here.
if arg.F:
func_name = arg.F
func = cmd_ev.procs.get(func_name)
if func is None:
raise error.Usage('Function %r not found' % func_name)
actions.append(
completion.ShellFuncAction(cmd_ev, func, self.comp_lookup))
# NOTE: We need completion for -A action itself!!! bash seems to have it.
for name in arg.actions:
if name == 'alias':
a = _FixedWordsAction(self.parse_ctx.aliases)
elif name == 'binding':
# TODO: Where do we get this from?
a = _FixedWordsAction(['vi-delete'])
elif name == 'command':
# compgen -A command in bash is SIX things: aliases, builtins,
# functions, keywords, external commands relative to the current
# directory, and external commands in $PATH.
actions.append(_FixedWordsAction(consts.BUILTIN_NAMES))
actions.append(_FixedWordsAction(self.parse_ctx.aliases))
actions.append(_FixedWordsAction(cmd_ev.procs))
actions.append(_FixedWordsAction(lexer_def.OSH_KEYWORD_NAMES))
actions.append(completion.FileSystemAction(exec_only=True))
# Look on the file system.
a = completion.ExternalCommandAction(cmd_ev.mem)
elif name == 'directory':
a = completion.FileSystemAction(dirs_only=True)
elif name == 'file':
a = completion.FileSystemAction()
elif name == 'function':
a = _FixedWordsAction(cmd_ev.procs)
elif name == 'job':
a = _FixedWordsAction(['jobs-not-implemented'])
elif name == 'user':
a = completion.UsersAction()
elif name == 'variable':
a = completion.VariablesAction(cmd_ev.mem)
elif name == 'helptopic':
# Note: it would be nice to have 'helpgroup' for help -i too
a = _FixedWordsAction(help_.TOPICS)
elif name == 'setopt':
names = [
opt.name for opt in option_def.All()
if opt.builtin == 'set'
]
a = _FixedWordsAction(names)
elif name == 'shopt':
names = [
opt.name for opt in option_def.All()
if opt.builtin == 'shopt'
]
a = _FixedWordsAction(names)
elif name == 'signal':
a = _FixedWordsAction(['TODO:signals'])
elif name == 'stopped':
a = _FixedWordsAction(['jobs-not-implemented'])
else:
raise NotImplementedError(name)
actions.append(a)
# e.g. -W comes after -A directory
if arg.W is not None: # could be ''
# NOTES:
# - Parsing is done at REGISTRATION time, but execution and splitting is
# done at COMPLETION time (when the user hits tab). So parse errors
# happen early.
w_parser = self.parse_ctx.MakeWordParserForPlugin(arg.W)
arena = self.parse_ctx.arena
try:
arg_word = w_parser.ReadForPlugin()
except error.Parse as e:
ui.PrettyPrintError(e, arena)
raise # Let 'complete' or 'compgen' return 2
a = completion.DynamicWordsAction(self.word_ev, self.splitter,
arg_word, arena)
actions.append(a)
extra_actions = []
if base_opts.get('plusdirs'):
extra_actions.append(completion.FileSystemAction(dirs_only=True))
# These only happen if there were zero shown.
else_actions = []
if base_opts.get('default'):
else_actions.append(completion.FileSystemAction())
if base_opts.get('dirnames'):
else_actions.append(completion.FileSystemAction(dirs_only=True))
if not actions and not else_actions:
raise error.Usage('No actions defined in completion: %s' % argv)
p = completion.DefaultPredicate
if arg.X:
filter_pat = arg.X
if filter_pat.startswith('!'):
p = completion.GlobPredicate(False, filter_pat[1:])
else:
p = completion.GlobPredicate(True, filter_pat)
return completion.UserSpec(actions,
extra_actions,
else_actions,
p,
prefix=arg.P or '',
suffix=arg.S or '')
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cpp-consts':
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
LIST_INT = [
'STRICT_ALL',
'OIL_BASIC',
'OIL_ALL',
'DEFAULT_TRUE',
'PARSE_OPTION_NUMS',
'SHOPT_OPTION_NUMS',
'SET_OPTION_NUMS',
]
# TODO: These could be changed to numbers
LIST_STR = ['VISIBLE_SHOPT_NAMES']
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef CONSTS_H
#define CONSTS_H
#include "mylib.h"
#include "id_kind_asdl.h"
#include "option_asdl.h"
#include "runtime_asdl.h"
#include "types_asdl.h"
namespace consts {
""")
for name in LIST_INT:
out('extern List<int>* %s;', name)
for name in LIST_STR:
out('extern List<Str*>* %s;', name)
out("""\
extern int NO_INDEX;
int RedirDefaultFd(id_kind_asdl::Id_t id);
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
option_asdl::builtin_t LookupNormalBuiltin(Str* s);
option_asdl::builtin_t LookupAssignBuiltin(Str* s);
option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
bool IsControlFlow(Str* s);
bool IsKeyword(Str* s);
Str* LookupCharC(Str* c);
Str* LookupCharPrompt(Str* c);
Str* OptionName(option_asdl::option_t opt_num);
Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
} // namespace consts
#endif // CONSTS_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "consts.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
using types_asdl::redir_arg_type_e;
using types_asdl::bool_arg_type_e;
using option_asdl::builtin_t;
namespace consts {
int NO_INDEX = 0; // duplicated from frontend/consts.py
""")
# Note: could use opt_num:: instead of raw ints
for name in LIST_INT:
val = getattr(consts, name)
val_str = ', '.join(str(i) for i in val)
out('List<int>* %s = new List<int>({%s});', name, val_str)
for name in LIST_STR:
val = getattr(consts, name)
val_str = '/* TODO */'
out('List<Str*>* %s = new List<Str*>({%s});', name, val_str)
out("""\
int RedirDefaultFd(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_DEFAULT_FD):
a = Id_str(id_).replace('.', '::')
b = consts.REDIR_DEFAULT_FD[id_]
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# redir_arg_type_e::Path, etc.
b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
'.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(BOOL_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# bool_arg_type_e::Str, etc.
b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
Kind GetKind(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
b = builtin_def.BuiltinDict()
GenBuiltinLookup(b, 'LookupNormalBuiltin', 'normal', f)
GenBuiltinLookup(b, 'LookupAssignBuiltin', 'assign', f)
GenBuiltinLookup(b, 'LookupSpecialBuiltin', 'special', f)
# TODO: Fill these in
out("""\
bool IsControlFlow(Str* s) {
assert(0);
}
bool IsKeyword(Str* s) {
assert(0);
}
""")
GenCharLookup('LookupCharC', consts._ONE_CHAR_C, f, required=True)
GenCharLookup('LookupCharPrompt', consts._ONE_CHAR_PROMPT, f)
# OptionName() is a bit redundant with ADSL's option_str(), but we can
# remove that.
out("""\
Str* OptionName(option_asdl::option_t opt_num) {
const char* s;
switch (opt_num) {
""")
# These are the only ones we use
set_opts = [(opt.index, opt.name) for opt in option_def.All()
if opt.builtin == 'set']
for index, name in set_opts:
out(' case %s:' % index)
out(' s = "%s";' % name)
out(' break;')
out("""\
default:
assert(0);
}
return new Str(s); // TODO-intern
}
""")
#
# Generate a tightly packed 2D array for C, from a Python dict.
#
edges = consts._IFS_EDGES
max_state = max(edge[0] for edge in edges)
max_char_kind = max(edge[1] for edge in edges)
edge_array = []
for i in xrange(max_state + 1):
# unused cells get -1
edge_array.append(['-1'] * (max_char_kind + 1))
for i in xrange(max_state + 1):
for j in xrange(max_char_kind + 1):
entry = edges.get((i, j))
if entry is not None:
# pack (new_state, action) into 32 bits
edge_array[i][j] = '(%d<<16)|%d' % entry
parts = []
for i in xrange(max_state + 1):
parts.append(' {')
parts.append(', '.join('%10s' % cell
for cell in edge_array[i]))
parts.append(' },\n')
out("""\
int _IFS_EDGE[%d][%d] = {
%s
};
""" % (max_state + 1, max_char_kind + 1, ''.join(parts)))
out("""\
// Note: all of these are integers, e.g. state_i, emit_i, char_kind_i
using runtime_asdl::state_t;
using runtime_asdl::emit_t;
using runtime_asdl::char_kind_t;
Tuple2<state_t, emit_t> IfsEdge(state_t state, runtime_asdl::char_kind_t ch) {
int cell = _IFS_EDGE[state][ch];
state_t new_state = cell >> 16;
emit_t emit = cell & 0xFFFF;
return Tuple2<state_t, emit_t>(new_state, emit);
}
""")
out("""\
} // namespace consts
""")
elif action == 'py-consts':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
if 0:
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(consts.REDIR_DEFAULT_FD):
v = consts.REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(consts.REDIR_ARG_TYPES):
v = consts.REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)
