def get_var_args_iterator(self): """ Yields a key/value pair, the key is None, if its not a named arg. """ def iterate(): # `var_args` is typically an Array, and not a list. for stmt in self.var_args: if not isinstance(stmt, pr.Statement): if stmt is None: yield None, None continue old = stmt # generate a statement if it's not already one. module = builtin.Builtin.scope stmt = pr.Statement(module, [], (0, 0), None) stmt._commands = [old] # *args commands = stmt.get_commands() if not len(commands): continue if commands[0] == '*': arrays = evaluate.follow_call_list(commands[1:]) # *args must be some sort of an array, otherwise -> ignore for array in arrays: if isinstance(array, Array): for field_stmt in array: # yield from plz! yield None, field_stmt elif isinstance(array, Generator): for field_stmt in array.iter_content(): yield None, helpers.FakeStatement(field_stmt) # **kwargs elif commands[0] == '**': arrays = evaluate.follow_call_list(commands[1:]) for array in arrays: if isinstance(array, Array): for key_stmt, value_stmt in array.items(): # first index, is the key if syntactically correct call = key_stmt.get_commands()[0] if isinstance(call, pr.Name): yield call, value_stmt elif isinstance(call, pr.Call): yield call.name, value_stmt # Normal arguments (including key arguments). else: if stmt.assignment_details: key_arr, op = stmt.assignment_details[0] # named parameter if key_arr and isinstance(key_arr[0], pr.Call): yield key_arr[0].name, stmt else: yield None, stmt return iter(common.PushBackIterator(iterate()))
def get_params(execution_context, var_args): result_params = [] param_dict = {} funcdef = execution_context.tree_node parent_context = execution_context.parent_context for param in funcdef.params: param_dict[param.name.value] = param unpacked_va = list(var_args.unpack(funcdef)) var_arg_iterator = common.PushBackIterator(iter(unpacked_va)) non_matching_keys = defaultdict(lambda: []) keys_used = {} keys_only = False had_multiple_value_error = False for param in funcdef.params: # The value and key can both be null. There, the defaults apply. # args / kwargs will just be empty arrays / dicts, respectively. # Wrong value count is just ignored. If you try to test cases that are # not allowed in Python, Jedi will maybe not show any completions. key, argument = next(var_arg_iterator, (None, None)) while key is not None: keys_only = True try: key_param = param_dict[key] except KeyError: non_matching_keys[key] = argument else: if key in keys_used: had_multiple_value_error = True m = ("TypeError: %s() got multiple values for keyword argument '%s'." % (funcdef.name, key)) for node in var_args.get_calling_nodes(): analysis.add(parent_context, 'type-error-multiple-values', node, message=m) else: keys_used[key] = ExecutedParam(execution_context, key_param, argument) key, argument = next(var_arg_iterator, (None, None)) try: result_params.append(keys_used[param.name.value]) continue except KeyError: pass if param.star_count == 1: # *args param lazy_context_list = [] if argument is not None: lazy_context_list.append(argument) for key, argument in var_arg_iterator: # Iterate until a key argument is found. if key: var_arg_iterator.push_back((key, argument)) break lazy_context_list.append(argument) seq = iterable.FakeSequence(execution_context.evaluator, 'tuple', lazy_context_list) result_arg = context.LazyKnownContext(seq) elif param.star_count == 2: # **kwargs param dct = iterable.FakeDict(execution_context.evaluator, dict(non_matching_keys)) result_arg = context.LazyKnownContext(dct) non_matching_keys = {} else: # normal param if argument is None: # No value: Return an empty container if param.default is None: result_arg = context.LazyUnknownContext() if not keys_only: for node in var_args.get_calling_nodes(): m = _error_argument_count(funcdef, len(unpacked_va)) analysis.add(parent_context, 'type-error-too-few-arguments', node, message=m) else: result_arg = context.LazyTreeContext(parent_context, param.default) else: result_arg = argument result_params.append(ExecutedParam(execution_context, param, result_arg)) if not isinstance(result_arg, context.LazyUnknownContext): keys_used[param.name.value] = result_params[-1] if keys_only: # All arguments should be handed over to the next function. It's not # about the values inside, it's about the names. Jedi needs to now that # there's nothing to find for certain names. for k in set(param_dict) - set(keys_used): param = param_dict[k] if not (non_matching_keys or had_multiple_value_error or param.star_count or param.default): # add a warning only if there's not another one. for node in var_args.get_calling_nodes(): m = _error_argument_count(funcdef, len(unpacked_va)) analysis.add(parent_context, 'type-error-too-few-arguments', node, message=m) for key, lazy_context in non_matching_keys.items(): m = "TypeError: %s() got an unexpected keyword argument '%s'." \ % (funcdef.name, key) add_argument_issue( parent_context, 'type-error-keyword-argument', lazy_context, message=m ) remaining_arguments = list(var_arg_iterator) if remaining_arguments: m = _error_argument_count(funcdef, len(unpacked_va)) # Just report an error for the first param that is not needed (like # cPython). first_key, lazy_context = remaining_arguments[0] if var_args.get_calling_nodes(): # There might not be a valid calling node so check for that first. add_argument_issue(parent_context, 'type-error-too-many-arguments', lazy_context, message=m) return result_params
def get_params(evaluator, func, var_args): def gen_param_name_copy(param, keys=(), values=(), array_type=None): """ Create a param with the original scope (of varargs) as parent. """ if isinstance(var_args, pr.Array): parent = var_args.parent start_pos = var_args.start_pos else: parent = func start_pos = 0, 0 new_param = copy.copy(param) new_param.is_generated = True if parent is not None: new_param.parent = parent # create an Array (-> needed for *args/**kwargs tuples/dicts) arr = pr.Array(helpers.FakeSubModule, start_pos, array_type, parent) arr.values = values key_stmts = [] for key in keys: key_stmts.append(helpers.FakeStatement([key], start_pos)) arr.keys = key_stmts arr.type = array_type new_param._expression_list = [arr] name = copy.copy(param.get_name()) name.parent = new_param return name result = [] start_offset = 0 from jedi.evaluate.representation import InstanceElement if isinstance(func, InstanceElement): # Care for self -> just exclude it and add the instance start_offset = 1 self_name = copy.copy(func.params[0].get_name()) self_name.parent = func.instance result.append(self_name) param_dict = {} for param in func.params: param_dict[str(param.get_name())] = param # There may be calls, which don't fit all the params, this just ignores it. var_arg_iterator = common.PushBackIterator( _var_args_iterator(evaluator, var_args)) non_matching_keys = [] keys_used = set() keys_only = False for param in func.params[start_offset:]: # The value and key can both be null. There, the defaults apply. # args / kwargs will just be empty arrays / dicts, respectively. # Wrong value count is just ignored. If you try to test cases that are # not allowed in Python, Jedi will maybe not show any completions. key, value = next(var_arg_iterator, (None, None)) while key: keys_only = True try: key_param = param_dict[str(key)] except KeyError: non_matching_keys.append((key, value)) else: keys_used.add(str(key)) result.append(gen_param_name_copy(key_param, values=[value])) key, value = next(var_arg_iterator, (None, None)) expression_list = param.expression_list() keys = [] values = [] array_type = None ignore_creation = False if expression_list[0] == '*': # *args param array_type = pr.Array.TUPLE if value: values.append(value) for key, value in var_arg_iterator: # Iterate until a key argument is found. if key: var_arg_iterator.push_back((key, value)) break values.append(value) elif expression_list[0] == '**': # **kwargs param array_type = pr.Array.DICT if non_matching_keys: keys, values = zip(*non_matching_keys) elif not keys_only: # normal param if value is not None: values = [value] else: if param.assignment_details: # No value: return the default values. ignore_creation = True result.append(param.get_name()) param.is_generated = True else: # If there is no assignment detail, that means there is no # assignment, just the result. Therefore nothing has to be # returned. values = [] # Just ignore all the params that are without a key, after one keyword # argument was set. if not ignore_creation and (not keys_only or expression_list[0] == '**'): keys_used.add(str(key)) result.append( gen_param_name_copy(param, keys=keys, values=values, array_type=array_type)) if keys_only: # sometimes param arguments are not completely written (which would # create an Exception, but we have to handle that). for k in set(param_dict) - keys_used: result.append(gen_param_name_copy(param_dict[k])) return result
def get_params(evaluator, func, var_args): param_names = [] param_dict = {} for param in func.params: param_dict[str(param.get_name())] = param unpacked_va = list(var_args.unpack(func)) from jedi.evaluate.representation import InstanceElement if isinstance(func, InstanceElement): # Include self at this place. unpacked_va.insert( 0, (None, [iterable.AlreadyEvaluated([func.instance])])) var_arg_iterator = common.PushBackIterator(iter(unpacked_va)) non_matching_keys = defaultdict(lambda: []) keys_used = {} keys_only = False had_multiple_value_error = False for param in func.params: # The value and key can both be null. There, the defaults apply. # args / kwargs will just be empty arrays / dicts, respectively. # Wrong value count is just ignored. If you try to test cases that are # not allowed in Python, Jedi will maybe not show any completions. default = [] if param.default is None else [param.default] key, va_values = next(var_arg_iterator, (None, default)) while key is not None: keys_only = True k = unicode(key) try: key_param = param_dict[unicode(key)] except KeyError: non_matching_keys[key] += va_values else: param_names.append( ExecutedParam(key_param, var_args, va_values).name) if k in keys_used: had_multiple_value_error = True m = ( "TypeError: %s() got multiple values for keyword argument '%s'." % (func.name, k)) calling_va = _get_calling_var_args(evaluator, var_args) if calling_va is not None: analysis.add(evaluator, 'type-error-multiple-values', calling_va, message=m) else: try: keys_used[k] = param_names[-1] except IndexError: # TODO this is wrong stupid and whatever. pass key, va_values = next(var_arg_iterator, (None, ())) values = [] if param.stars == 1: # *args param lst_values = [iterable.MergedNodes(va_values)] if va_values else [] for key, va_values in var_arg_iterator: # Iterate until a key argument is found. if key: var_arg_iterator.push_back((key, va_values)) break if va_values: lst_values.append(iterable.MergedNodes(va_values)) seq = iterable.FakeSequence(evaluator, lst_values, 'tuple') values = [iterable.AlreadyEvaluated([seq])] elif param.stars == 2: # **kwargs param dct = iterable.FakeDict(evaluator, dict(non_matching_keys)) values = [iterable.AlreadyEvaluated([dct])] non_matching_keys = {} else: # normal param if va_values: values = va_values else: # No value: Return an empty container values = [] if not keys_only: calling_va = var_args.get_calling_var_args() if calling_va is not None: m = _error_argument_count(func, len(unpacked_va)) analysis.add(evaluator, 'type-error-too-few-arguments', calling_va, message=m) # Now add to result if it's not one of the previously covered cases. if (not keys_only or param.stars == 2): param_names.append(ExecutedParam(param, var_args, values).name) keys_used[unicode(param.get_name())] = param_names[-1] if keys_only: # All arguments should be handed over to the next function. It's not # about the values inside, it's about the names. Jedi needs to now that # there's nothing to find for certain names. for k in set(param_dict) - set(keys_used): param = param_dict[k] values = [] if param.default is None else [param.default] param_names.append(ExecutedParam(param, var_args, values).name) if not (non_matching_keys or had_multiple_value_error or param.stars or param.default): # add a warning only if there's not another one. calling_va = _get_calling_var_args(evaluator, var_args) if calling_va is not None: m = _error_argument_count(func, len(unpacked_va)) analysis.add(evaluator, 'type-error-too-few-arguments', calling_va, message=m) for key, va_values in non_matching_keys.items(): m = "TypeError: %s() got an unexpected keyword argument '%s'." \ % (func.name, key) for value in va_values: analysis.add(evaluator, 'type-error-keyword-argument', value.parent, message=m) remaining_params = list(var_arg_iterator) if remaining_params: m = _error_argument_count(func, len(unpacked_va)) # Just report an error for the first param that is not needed (like # cPython). first_key, first_values = remaining_params[0] for v in first_values: if first_key is not None: # Is a keyword argument, return the whole thing instead of just # the value node. v = v.parent try: non_kw_param = keys_used[first_key] except KeyError: pass else: origin_args = non_kw_param.parent.var_args.argument_node # TODO calculate the var_args tree and check if it's in # the tree (if not continue). # print('\t\tnonkw', non_kw_param.parent.var_args.argument_node, ) if origin_args not in [ f.parent.parent for f in first_values ]: continue analysis.add(evaluator, 'type-error-too-many-arguments', v, message=m) return param_names
def _parse_statement(self): """ This is not done in the main parser, because it might be slow and most of the statements won't need this data anyway. This is something 'like' a lazy execution. This is not really nice written, sorry for that. If you plan to replace it and make it nicer, that would be cool :-) """ def is_assignment(tok): return isinstance(tok, (str, unicode)) and tok.endswith('=') \ and not tok in ['>=', '<=', '==', '!='] def parse_array(token_iterator, array_type, start_pos, add_el=None, added_breaks=()): arr = Array(self._sub_module, start_pos, array_type, self) if add_el is not None: arr.add_statement(add_el) maybe_dict = array_type == Array.SET break_tok = None is_array = None while True: stmt, break_tok = parse_stmt(token_iterator, maybe_dict, break_on_assignment=bool(add_el), added_breaks=added_breaks) if stmt is None: break else: if break_tok == ',': is_array = True is_key = maybe_dict and break_tok == ':' arr.add_statement(stmt, is_key) if break_tok in closing_brackets \ or break_tok in added_breaks \ or is_assignment(break_tok): break if arr.type == Array.TUPLE and len(arr) == 1 and not is_array: arr.type = Array.NOARRAY if not arr.values and maybe_dict: # this is a really special case - empty brackets {} are # always dictionaries and not sets. arr.type = Array.DICT c = token_iterator.current[1] arr.end_pos = c.end_pos if isinstance(c, Simple) \ else (c[2][0], c[2][1] + len(c[1])) return arr, break_tok def parse_stmt(token_iterator, maybe_dict=False, added_breaks=(), break_on_assignment=False, stmt_class=Statement): token_list = [] used_vars = [] level = 1 tok = None first = True end_pos = None, None for i, tok_temp in token_iterator: if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed tok = tok_temp if first: start_pos = tok.start_pos first = False end_pos = tok.end_pos if isinstance(tok, ListComprehension): # it's not possible to set it earlier tok.parent = self if isinstance(tok, Name): used_vars.append(tok) else: token_type, tok, start_tok_pos = tok_temp last_end_pos = end_pos end_pos = start_tok_pos[0], start_tok_pos[1] + len(tok) if first: first = False start_pos = start_tok_pos if tok == 'lambda': lambd, tok = parse_lambda(token_iterator) if lambd is not None: token_list.append(lambd) elif tok == 'for': list_comp, tok = parse_list_comp(token_iterator, token_list, start_pos, last_end_pos) if list_comp is not None: token_list = [list_comp] if tok in closing_brackets: level -= 1 elif tok in brackets.keys(): level += 1 if level == 0 and tok in closing_brackets \ or tok in added_breaks \ or level == 1 and (tok == ',' or maybe_dict and tok == ':' or is_assignment(tok) and break_on_assignment): end_pos = end_pos[0], end_pos[1] - 1 break token_list.append(tok_temp) if not token_list: return None, tok statement = stmt_class(self._sub_module, [], [], token_list, start_pos, end_pos, self.parent) statement.used_vars = used_vars return statement, tok def parse_lambda(token_iterator): params = [] start_pos = self.start_pos while True: param, tok = parse_stmt(token_iterator, added_breaks=[':'], stmt_class=Param) if param is None: break params.append(param) if tok == ':': break if tok != ':': return None, tok # since lambda is a Function scope, it needs Scope parents parent = self.get_parent_until(IsScope) lambd = Lambda(self._sub_module, params, start_pos, parent) ret, tok = parse_stmt(token_iterator) if ret is not None: ret.parent = lambd lambd.returns.append(ret) lambd.end_pos = self.end_pos return lambd, tok def parse_list_comp(token_iterator, token_list, start_pos, end_pos): def parse_stmt_or_arr(token_iterator, added_breaks=()): stmt, tok = parse_stmt(token_iterator, added_breaks=added_breaks) if not stmt: return None, tok if tok == ',': arr, tok = parse_array(token_iterator, Array.TUPLE, stmt.start_pos, stmt, added_breaks=added_breaks) used_vars = [] for stmt in arr: used_vars += stmt.used_vars start_pos = arr.start_pos[0], arr.start_pos[1] - 1 stmt = Statement(self._sub_module, [], used_vars, [], start_pos, arr.end_pos) arr.parent = stmt stmt.token_list = stmt._commands = [arr] else: for v in stmt.used_vars: v.parent = stmt return stmt, tok st = Statement(self._sub_module, [], [], token_list, start_pos, end_pos) middle, tok = parse_stmt_or_arr(token_iterator, added_breaks=['in']) if tok != 'in' or middle is None: debug.warning('list comprehension middle @%s' % str(start_pos)) return None, tok in_clause, tok = parse_stmt_or_arr(token_iterator) if in_clause is None: debug.warning('list comprehension in @%s' % str(start_pos)) return None, tok return ListComprehension(st, middle, in_clause, self), tok # initializations result = [] is_chain = False brackets = {'(': Array.TUPLE, '[': Array.LIST, '{': Array.SET} closing_brackets = ')', '}', ']' token_iterator = common.PushBackIterator(enumerate(self.token_list)) for i, tok_temp in token_iterator: if isinstance(tok_temp, Base): # the token is a Name, which has already been parsed tok = tok_temp token_type = None start_pos = tok.start_pos end_pos = tok.end_pos else: token_type, tok, start_pos = tok_temp end_pos = start_pos[0], start_pos[1] + len(tok) if is_assignment(tok): # This means, there is an assignment here. # Add assignments, which can be more than one self._assignment_details.append((result, tok)) result = [] is_chain = False continue elif tok == 'as': # just ignore as, because it sets values next(token_iterator, None) continue if tok == 'lambda': lambd, tok = parse_lambda(token_iterator) if lambd is not None: result.append(lambd) else: continue is_literal = token_type in [tokenize.STRING, tokenize.NUMBER] if isinstance(tok, Name) or is_literal: c_type = Call.NAME if is_literal: tok = literal_eval(tok) if token_type == tokenize.STRING: c_type = Call.STRING elif token_type == tokenize.NUMBER: c_type = Call.NUMBER call = Call(self._sub_module, tok, c_type, start_pos, end_pos, self) if is_chain: result[-1].set_next(call) else: result.append(call) is_chain = False elif tok in brackets.keys(): arr, is_ass = parse_array(token_iterator, brackets[tok], start_pos) if result and isinstance(result[-1], Call): result[-1].set_execution(arr) else: arr.parent = self result.append(arr) elif tok == '.': if result and isinstance(result[-1], Call): is_chain = True elif tok == ',': # implies a tuple # commands is now an array not a statement anymore t = result[0] start_pos = t[2] if isinstance(t, tuple) else t.start_pos # get the correct index i, tok = next(token_iterator, (len(self.token_list), None)) if tok is not None: token_iterator.push_back((i, tok)) t = self.token_list[i - 1] try: e = t.end_pos except AttributeError: e = (t[2][0], t[2][1] + len(t[1])) \ if isinstance(t, tuple) else t.start_pos stmt = Statement(self._sub_module, [], [], result, start_pos, e, self.parent) stmt._commands = result arr, break_tok = parse_array(token_iterator, Array.TUPLE, stmt.start_pos, stmt) result = [arr] if is_assignment(break_tok): self._assignment_details.append((result, break_tok)) result = [] is_chain = False else: if tok != '\n' and token_type != tokenize.COMMENT: result.append(tok) return result
def get_params(evaluator, func, var_args): result = [] param_dict = {} for param in func.params: param_dict[str(param.get_name())] = param # There may be calls, which don't fit all the params, this just ignores it. unpacked_va = _unpack_var_args(evaluator, var_args, func) var_arg_iterator = common.PushBackIterator(iter(unpacked_va)) non_matching_keys = [] keys_used = set() keys_only = False va_values = None had_multiple_value_error = False for param in func.params: # The value and key can both be null. There, the defaults apply. # args / kwargs will just be empty arrays / dicts, respectively. # Wrong value count is just ignored. If you try to test cases that are # not allowed in Python, Jedi will maybe not show any completions. key, va_values = next(var_arg_iterator, (None, [])) while key: keys_only = True k = unicode(key) try: key_param = param_dict[unicode(key)] except KeyError: non_matching_keys.append((key, va_values)) else: result.append(_gen_param_name_copy(func, var_args, key_param, values=va_values)) if k in keys_used: had_multiple_value_error = True m = ("TypeError: %s() got multiple values for keyword argument '%s'." % (func.name, k)) calling_va = _get_calling_var_args(evaluator, var_args) if calling_va is not None: analysis.add(evaluator, 'type-error-multiple-values', calling_va, message=m) else: keys_used.add(k) key, va_values = next(var_arg_iterator, (None, [])) keys = [] values = [] array_type = None has_default_value = False if param.stars == 1: # *args param array_type = pr.Array.TUPLE lst_values = [va_values] for key, va_values in var_arg_iterator: # Iterate until a key argument is found. if key: var_arg_iterator.push_back((key, va_values)) break lst_values.append(va_values) if lst_values[0]: values = [helpers.stmts_to_stmt(v) for v in lst_values] elif param.stars == 2: # **kwargs param array_type = pr.Array.DICT if non_matching_keys: keys, values = zip(*non_matching_keys) values = [helpers.stmts_to_stmt(list(v)) for v in values] non_matching_keys = [] else: # normal param if va_values: values = va_values else: if param.assignment_details: # No value: Return the default values. has_default_value = True result.append(param.get_name()) # TODO is this allowed? it changes it long time. param.is_generated = True else: # No value: Return an empty container values = [] if not keys_only and isinstance(var_args, pr.Array): calling_va = _get_calling_var_args(evaluator, var_args) if calling_va is not None: m = _error_argument_count(func, len(unpacked_va)) analysis.add(evaluator, 'type-error-too-few-arguments', calling_va, message=m) # Now add to result if it's not one of the previously covered cases. if not has_default_value and (not keys_only or param.stars == 2): keys_used.add(unicode(param.get_name())) result.append(_gen_param_name_copy(func, var_args, param, keys=keys, values=values, array_type=array_type)) if keys_only: # All arguments should be handed over to the next function. It's not # about the values inside, it's about the names. Jedi needs to now that # there's nothing to find for certain names. for k in set(param_dict) - keys_used: param = param_dict[k] result.append(_gen_param_name_copy(func, var_args, param)) if not (non_matching_keys or had_multiple_value_error or param.stars or param.assignment_details): # add a warning only if there's not another one. calling_va = _get_calling_var_args(evaluator, var_args) if calling_va is not None: m = _error_argument_count(func, len(unpacked_va)) analysis.add(evaluator, 'type-error-too-few-arguments', calling_va, message=m) for key, va_values in non_matching_keys: m = "TypeError: %s() got an unexpected keyword argument '%s'." \ % (func.name, key) for value in va_values: analysis.add(evaluator, 'type-error-keyword-argument', value, message=m) remaining_params = list(var_arg_iterator) if remaining_params: m = _error_argument_count(func, len(unpacked_va)) for p in remaining_params[0][1]: analysis.add(evaluator, 'type-error-too-many-arguments', p, message=m) return result