def vwrite_out(h, k, v_in):
if f_attr2is_appendable(k):
v_new = " ".join(
lchain(
h.get(k, "").split(),
v_in.split(),
))
return DictToolkit.VWrite.overwrite(h, k, v_new)
return vwrite_in(h, k, v_in)
def rootname_func2logger(cls, rootname, func, config=None):
from foxylib.tools.collections.itertools_tools import lchain
name = ".".join(
lchain([rootname],
FunctionToolkit.func2class_func_name_list(func)))
logger_raw = logging.getLogger(name)
if config:
logger = LoggerToolkit.config2logger(logger_raw, config=config)
else:
logger = logger_raw
return logger
def lexer_args(cls):
s_delim_eaters = ","
s_ANY_EXs = lchain(s_delim_eaters, DelimLexer.specialchars())
r_TUPLE = (
r"\s+",
r"(?:(?:[^\s{0}])|(?:{1}))".format(
"".join(map(re.escape, s_ANY_EXs)),
r"\s*(?:{0})\s*".format("|".join(map(re.escape,
s_delim_eaters))),
),
)
lexer = cls.r_TUPLE2lexer(r_TUPLE)
return lexer
def iter2iList_duplicates(
cls,
iterable,
key=None,
):
from foxylib.tools.collections.itertools_tools import lchain
if key is None:
key = lambda x: x
l_IN = list(iterable)
h = OrderedDict()
for i, x in enumerate(l_IN):
k = key(x)
h[k] = lchain(h.get(k, []), [i])
return lchain.from_iterable(filter(lambda l: len(l) > 1, h.values()))
class ParameterLexer(object):
# List of token names. This is always required
tokens = (
'ANY',
#'WHITESPACE',
'DQ',
'SQ',
'BACKSLASH',
'EQUAL',
'DELIM_PREFIX',
)
l_PREFIX = ["+", "-", "*", "?"]
t_DQ = r'"'
t_SQ = r"'"
t_BACKSLASH = r'\\'
t_EQUAL = r'='
t_DELIM_PREFIX = r'[{0}]'.format("".join(lmap(re.escape, l_PREFIX)))
l_VAR = lchain(
l_PREFIX,
[
"\\",
"=",
"'",
'"',
],
)
t_ANY = r'(?:[^\s{0}]+)|(?:\s+)'.format("".join(lmap(re.escape, l_VAR)))
#t_WHITESPACE = r'\W+'
# Error handling rule
def t_error(self, t):
raise Exception("Illegal character '%s'" % t.value[0])
# Build the lexer
def build(self, **kwargs):
self.lexer = lex.lex(module=self, **kwargs)
DELIM_TYPE_PREFIX = "P"
DELIM_TYPE_INFIX = "I"
@classmethod
def r_prefix(cls):
return r"(?:{0})".format("|".join(lmap(re.escape, cls.l_PREFIX)))
@classmethod
def stt_delim2type(cls, tok):
is_PREFIX = (tok.value in cls.l_PREFIX)
is_INFIX = (tok.value in ["="])
if len(lfilter(bool, [is_PREFIX, is_INFIX])) != 1: raise Exception()
if is_PREFIX: return cls.DELIM_TYPE_PREFIX
if is_INFIX: return cls.DELIM_TYPE_INFIX
raise Exception()
@classmethod
def delim_infix2iStart(
cls,
token_list_DELIM,
tt_list_DELIM,
):
if not token_list_DELIM: return None
tok_LAST = token_list_DELIM[-1]
if tok_LAST.type != "ANY": return None
if len(token_list_DELIM) <= 1: return -1
tok_2PREV = token_list_DELIM[-2]
if tok_2PREV.type not in tt_list_DELIM: return -1
delim_type = cls.stt_delim2type(tok_2PREV)
if delim_type == cls.DELIM_TYPE_INFIX: return None # Wrong syntax
if delim_type == cls.DELIM_TYPE_PREFIX: return -2
raise Exception()
@classmethod
def is_delim_infix_valid(cls, token_list_DELIM):
if not token_list_DELIM: return False
tok_LAST = token_list_DELIM[-1]
if tok_LAST.type != "ANY": return False
return True
@classmethod
def lexer2str_DELIM_list(
cls,
lexer,
s_IN,
maxsplit=None,
):
# delim_rule = LexerToolkit.DELIM_AS_PREFIX
lexer.input(s_IN)
tt_list_ESCAPE = ["BACKSLASH"]
tt_list_STATE = ["SQ", "DQ"]
tt_list_DELIM = [
"DELIM_PREFIX",
"EQUAL",
]
str_DELIM_list = []
token_list_DELIM = []
state_INITIAL = "INITIAL"
l_state = [
state_INITIAL,
]
while True:
tok = lexer.token()
if not tok: break
#print(tok, tok.type, file=sys.stderr)
state_CUR = l_state[-1]
stop_split = (maxsplit
is not None) and (len(str_DELIM_list) >= maxsplit)
stt = LexerToolkit.tok2semantic_token_type(
tok,
token_list_DELIM,
[
tt_list_ESCAPE,
tt_list_STATE,
tt_list_DELIM,
],
stop_split,
state_CUR,
state_INITIAL,
)
is_append_BEFORE = stt not in [LexerToolkit.STT_DELIM]
is_append_BEFORE_and_done = (stt in [LexerToolkit.STT_ANY])
if is_append_BEFORE: token_list_DELIM.append(tok)
if is_append_BEFORE_and_done: continue
if stt == LexerToolkit.STT_DELIM:
delim_type = cls.stt_delim2type(tok)
if delim_type == cls.DELIM_TYPE_INFIX:
iSTART_INFIX = cls.delim_infix2iStart(
token_list_DELIM,
tt_list_DELIM,
)
if iSTART_INFIX is None:
#raise Exception()
return None # Syntactically wrong
if iSTART_INFIX < -1:
if len(token_list_DELIM) != 2: raise Exception()
else:
token_list_PREV = token_list_DELIM[:iSTART_INFIX]
str_DELIM_list.append(
LexerToolkit.token_list_DELIM2str_DELIM(
token_list_PREV))
token_list_DELIM = token_list_DELIM[iSTART_INFIX:]
#print(tok, token_list_DELIM, str_DELIM_list, iSTART_INFIX, file=sys.stderr)
elif delim_type == cls.DELIM_TYPE_PREFIX:
token_list_PREV = token_list_DELIM
str_DELIM_list.append(
LexerToolkit.token_list_DELIM2str_DELIM(
token_list_PREV))
token_list_DELIM = []
else:
raise Exception()
token_list_DELIM.append(tok)
continue
if stt == LexerToolkit.STT_START:
l_state.append(tok.type)
continue
if stt == LexerToolkit.STT_END:
if l_state[-1] != tok.type: raise Exception()
l_state.pop()
continue
if len(l_state) > 1: return None
if l_state[0] != state_INITIAL: return None
if token_list_DELIM:
str_DELIM_list.append(
LexerToolkit.token_list_DELIM2str_DELIM(token_list_DELIM))
return str_DELIM_list
@classmethod
def str2l_token(
cls,
s,
maxsplit=None,
include_tokens=None,
):
if include_tokens is None: include_tokens = True
m = cls()
m.build()
tok_groups = (
["ANY", "SINGLEQUOTE", "DOUBLEQUOTE"],
["DELIM"],
)
l = LexerToolkit.str2str_token_list(
m.lexer,
s,
tok_groups,
maxsplit=maxsplit,
include_tokens=include_tokens,
)
return l
@classmethod
def create_lexer(cls):
m = cls()
m.build()
lexer = m.lexer
return lexer
@classmethod
def str2args_kwargs_pair(
cls,
s_IN,
maxsplit=None,
):
str_PARAM_list = cls.lexer2str_DELIM_list(
cls.create_lexer(),
s_IN,
maxsplit=maxsplit,
)
if not str_PARAM_list: return (None, str_PARAM_list)
return (
str2strip(str_PARAM_list[0]),
lmap(str2strip, str_PARAM_list[1:]),
)
@classmethod
def str2args_kwargs_pair_NEW(
cls,
s_IN,
split_ARG_str,
):
str_PARAM_list = cls.lexer2str_DELIM_list(
cls.create_lexer(),
s_IN,
)
if not str_PARAM_list: return (None, str_PARAM_list)
return (
str2strip(str_PARAM_list[0]),
lmap(str2strip, str_PARAM_list[1:]),
)