def check(self):
return not search_ast(
self.tree,
ast.BinOp(left=ast.Str(), op=ast.Add(), right=ast.Str()),
)
def ast_raise_type_err(msg: str, data_var: BlockLevel):
return ast.Raise(
ast_call(
ast_name('_TypeError'),
[ast_call(ast_attr(ast.Str(msg), 'format'), [ast_name(data_var)])
]), None)
def _ast_for_value(v):
if isinstance(v, (str, unicode)): return ast.Str(s=v)
elif isinstance(v, int): return ast.Num(n=v)
else: raise NotImplementedError("type (%r) %r" % (type(v), v))
def __init__(self):
super().__init__()
cast(ast.Call, self.query_ast).args.append(ast.Str(s="hi"))
def mutate_Str_empty(self, node):
if not node.s or utils.is_docstring(node):
raise MutationResign()
return ast.Str(s='')
def visit_Str(self, tree):
return ast.Str("String: " + tree.s)
def visit_Assert(self, assert_: ast.Assert) -> List[ast.stmt]:
"""Return the AST statements to replace the ast.Assert instance.
This rewrites the test of an assertion to provide
intermediate values and replace it with an if statement which
raises an assertion error with a detailed explanation in case
the expression is false.
"""
if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1:
from _pytest.warning_types import PytestAssertRewriteWarning
import warnings
# TODO: This assert should not be needed.
assert self.module_path is not None
warnings.warn_explicit(
PytestAssertRewriteWarning(
"assertion is always true, perhaps remove parentheses?"),
category=None,
filename=fspath(self.module_path),
lineno=assert_.lineno,
)
self.statements = [] # type: List[ast.stmt]
self.variables = [] # type: List[str]
self.variable_counter = itertools.count()
if self.enable_assertion_pass_hook:
self.format_variables = [] # type: List[str]
self.stack = [] # type: List[Dict[str, ast.expr]]
self.expl_stmts = [] # type: List[ast.stmt]
self.push_format_context()
# Rewrite assert into a bunch of statements.
top_condition, explanation = self.visit(assert_.test)
negation = ast.UnaryOp(ast.Not(), top_condition)
if self.enable_assertion_pass_hook: # Experimental pytest_assertion_pass hook
msg = self.pop_format_context(ast.Str(explanation))
# Failed
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
gluestr = "\n>assert "
else:
assertmsg = ast.Str("")
gluestr = "assert "
err_explanation = ast.BinOp(ast.Str(gluestr), ast.Add(), msg)
err_msg = ast.BinOp(assertmsg, ast.Add(), err_explanation)
err_name = ast.Name("AssertionError", ast.Load())
fmt = self.helper("_format_explanation", err_msg)
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
statements_fail = []
statements_fail.extend(self.expl_stmts)
statements_fail.append(raise_)
# Passed
fmt_pass = self.helper("_format_explanation", msg)
orig = self._assert_expr_to_lineno()[assert_.lineno]
hook_call_pass = ast.Expr(
self.helper(
"_call_assertion_pass",
ast.Num(assert_.lineno),
ast.Str(orig),
fmt_pass,
))
# If any hooks implement assert_pass hook
hook_impl_test = ast.If(
self.helper("_check_if_assertion_pass_impl"),
self.expl_stmts + [hook_call_pass],
[],
)
statements_pass = [hook_impl_test]
# Test for assertion condition
main_test = ast.If(negation, statements_fail, statements_pass)
self.statements.append(main_test)
if self.format_variables:
variables = [
ast.Name(name, ast.Store())
for name in self.format_variables
]
clear_format = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear_format)
else: # Original assertion rewriting
# Create failure message.
body = self.expl_stmts
self.statements.append(ast.If(negation, body, []))
if assert_.msg:
assertmsg = self.helper("_format_assertmsg", assert_.msg)
explanation = "\n>assert " + explanation
else:
assertmsg = ast.Str("")
explanation = "assert " + explanation
template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
msg = self.pop_format_context(template)
fmt = self.helper("_format_explanation", msg)
err_name = ast.Name("AssertionError", ast.Load())
exc = ast.Call(err_name, [fmt], [])
raise_ = ast.Raise(exc, None)
body.append(raise_)
# Clear temporary variables by setting them to None.
if self.variables:
variables = [
ast.Name(name, ast.Store()) for name in self.variables
]
clear = ast.Assign(variables, ast.NameConstant(None))
self.statements.append(clear)
# Fix line numbers.
for stmt in self.statements:
set_location(stmt, assert_.lineno, assert_.col_offset)
return self.statements
def visit_Str(self, node):
return ast.Str("str: " + node.s)
def _create_attributes_nodes(self, prepared, I18N_ATTRIBUTES):
attributes = []
for name, text, quote, space, eq, expr in prepared:
implicit_i18n = name.lower() in self.implicit_i18n_attributes
char_escape = ('&', '<', '>', quote)
# Use a provided default text as the default marker
# (aliased to the name ``default``), otherwise use the
# program's default marker value.
if text is not None:
default_marker = ast.Str(s=text)
else:
default_marker = self.default_marker
msgid = I18N_ATTRIBUTES.get(name, missing)
# If (by heuristic) ``text`` contains one or more
# interpolation expressions, apply interpolation
# substitution to the text
if expr is None and text is not None and '${' in text:
expr = nodes.Substitution(text, char_escape, None)
translation = implicit_i18n and msgid is missing
value = nodes.Interpolation(expr, True, translation)
default_marker = self.default_marker
# If the expression is non-trivial, the attribute is
# dynamic (computed).
elif expr is not None:
if name in self.boolean_attributes:
value = nodes.Boolean(expr, name)
else:
if text is not None:
default = default_marker
else:
default = None
value = nodes.Substitution(expr, char_escape, default)
# Otherwise, it's a static attribute.
else:
value = ast.Str(s=text)
if msgid is missing and implicit_i18n:
msgid = text
# If translation is required, wrap in a translation
# clause
if msgid is not missing:
value = nodes.Translate(msgid, value)
attribute = nodes.Attribute(name, value, quote, eq, space)
# Always define a ``default`` alias
attribute = nodes.Define(
[nodes.Alias(["default"], default_marker)],
attribute,
)
attributes.append(attribute)
return attributes
def visit_Assert(self, assert_):
"""Return the AST statements to replace the ast.Assert instance.
This rewrites the test of an assertion to provide
intermediate values and replace it with an if statement which
raises an assertion error with a detailed explanation in case
the expression is false.
"""
if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1:
from _pytest.warning_types import PytestWarning
import warnings
warnings.warn_explicit(
PytestWarning(
"assertion is always true, perhaps remove parentheses?"),
category=None,
filename=str(self.module_path),
lineno=assert_.lineno,
)
self.statements = []
self.variables = []
self.variable_counter = itertools.count()
self.stack = []
self.on_failure = []
self.push_format_context()
# Rewrite assert into a bunch of statements.
top_condition, explanation = self.visit(assert_.test)
# If in a test module, check if directly asserting None, in order to warn [Issue #3191]
if self.module_path is not None:
self.statements.append(
self.warn_about_none_ast(top_condition,
module_path=self.module_path,
lineno=assert_.lineno))
# Create failure message.
body = self.on_failure
negation = ast.UnaryOp(ast.Not(), top_condition)
self.statements.append(ast.If(negation, body, []))
if assert_.msg:
assertmsg = self.helper("format_assertmsg", assert_.msg)
explanation = "\n>assert " + explanation
else:
assertmsg = ast.Str("")
explanation = "assert " + explanation
template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation))
msg = self.pop_format_context(template)
fmt = self.helper("format_explanation", msg)
err_name = ast.Name("AssertionError", ast.Load())
exc = ast_Call(err_name, [fmt], [])
if sys.version_info[0] >= 3:
raise_ = ast.Raise(exc, None)
else:
raise_ = ast.Raise(exc, None, None)
body.append(raise_)
# Clear temporary variables by setting them to None.
if self.variables:
variables = [
ast.Name(name, ast.Store()) for name in self.variables
]
clear = ast.Assign(variables, _NameConstant(None))
self.statements.append(clear)
# Fix line numbers.
for stmt in self.statements:
set_location(stmt, assert_.lineno, assert_.col_offset)
return self.statements
def string_node(str):
return ast.Str(s=str)
def visit_Num(self, node): # noqa
"""Replace Num node to Decimal instance."""
return ast.Call(func=ast.Name(id='Decimal', ctx=ast.Load()),
args=[ast.Str(s=str(node.n))],
keywords=[])
def subscript(name, value, ctx):
return ast.Subscript(
value=value,
slice=ast.Index(value=ast.Str(s=name)),
ctx=ctx,
)
def marker(name):
return ast.Str(s="__%s" % name)
ast.arg(arg='self',
annotation=None,
lineno=8,
col_offset=17),
ast.arg(arg='provided_argument_string',
annotation=None,
lineno=8,
col_offset=23),
],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[
ast.Str(s='',
lineno=8,
col_offset=48),
]),
body=[
ast.Expr(value=ast.Call(
func=ast.Attribute(value=ast.Call(
func=ast.Name(id='super',
ctx=ast.Load(),
lineno=9,
col_offset=8),
args=[],
keywords=[],
lineno=9,
col_offset=8),
attr='__init__',
ctx=ast.Load(),
def visit_FunctionDef(self, node):
if node.name:
self.stack[-1].scope.declare(node.name)
args = [get_arg_id(arg) for arg in node.args.args]
if node.args.kwarg is not None:
raise Exception('**kwargs not supported')
func = FunctionContext(self.stack[-1], args)
self.push(func)
# Emit vararg
if node.args.vararg is not None:
self.visit(
ast.Assign([ast.Name(node.args.vararg, ast.Store())],
ast_call(
ast_load('Array.prototype.slice.call'),
ast.Name('arguments', ast.Load()),
ast.Num(len(args)),
)))
# Emit default arguments
def_args = node.args.defaults
for arg_name, arg_val in zip(args[-len(def_args):], def_args):
self.block([
ast.If(
ast.Compare(
ast_call(
ast.Name('type', ast.Load()),
ast.Name(arg_name, ast.Load()),
),
[
ast.Eq(),
],
[
ast.Str('undefined'),
],
),
[
ast.Assign([ast.Name(arg_name, ast.Store())], arg_val),
],
[],
)
])
# Emit function body
self.block(node.body)
body = ast_call(
ast_load('JS'),
ast.Str(str(self.stack.pop())),
)
for decorator in node.decorator_list:
body = ast_call(decorator, body)
if not node.name:
self.visit(body)
else:
self.visit(ast.Assign(
[ast_load(node.name)],
body,
))
def ast_string_node(string: str) -> ast.Str:
return ast.Str(s=string)
def _compile_directive_call_assets(self, el, options):
""" This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
if len(el):
raise SyntaxError("t-call-assets cannot contain children nodes")
# nodes = self._get_asset_nodes(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values)
#
# for index, (tagName, t_attrs, content) in enumerate(nodes):
# if index:
# append('\n ')
# append('<')
# append(tagName)
#
# self._post_processing_att(tagName, t_attrs, options)
# for name, value in t_attrs.items():
# if value or isinstance(value, string_types)):
# append(u' ')
# append(name)
# append(u'="')
# append(escape(pycompat.to_text((value)))
# append(u'"')
#
# if not content and tagName in self._void_elements:
# append('/>')
# else:
# append('>')
# if content:
# append(content)
# append('</')
# append(tagName)
# append('>')
#
space = el.getprevious() is not None and el.getprevious().tail or el.getparent().text
sep = u'\n' + space.rsplit('\n').pop()
return [
ast.Assign(
targets=[ast.Name(id='nodes', ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='_get_asset_nodes',
ctx=ast.Load()
),
args=[
ast.Str(el.get('t-call-assets')),
ast.Name(id='options', ctx=ast.Load()),
],
keywords=[
ast.keyword('css', self._get_attr_bool(el.get('t-css', True))),
ast.keyword('js', self._get_attr_bool(el.get('t-js', True))),
ast.keyword('debug', ast.Call(
func=ast.Attribute(
value=ast.Name(id='values', ctx=ast.Load()),
attr='get',
ctx=ast.Load()
),
args=[ast.Str('debug')],
keywords=[], starargs=None, kwargs=None
)),
ast.keyword('async_load', self._get_attr_bool(el.get('async_load', False))),
ast.keyword('defer_load', self._get_attr_bool(el.get('defer_load', False))),
ast.keyword('lazy_load', self._get_attr_bool(el.get('lazy_load', False))),
ast.keyword('values', ast.Name(id='values', ctx=ast.Load())),
],
starargs=None, kwargs=None
)
),
ast.For(
target=ast.Tuple(elts=[
ast.Name(id='index', ctx=ast.Store()),
ast.Tuple(elts=[
ast.Name(id='tagName', ctx=ast.Store()),
ast.Name(id='t_attrs', ctx=ast.Store()),
ast.Name(id='content', ctx=ast.Store())
], ctx=ast.Store())
], ctx=ast.Store()),
iter=ast.Call(
func=ast.Name(id='enumerate', ctx=ast.Load()),
args=[ast.Name(id='nodes', ctx=ast.Load())],
keywords=[],
starargs=None, kwargs=None
),
body=[
ast.If(
test=ast.Name(id='index', ctx=ast.Load()),
body=[self._append(ast.Str(sep))],
orelse=[]
),
self._append(ast.Str(u'<')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
] + self._append_attributes() + [
ast.If(
test=ast.BoolOp(
op=ast.And(),
values=[
ast.UnaryOp(ast.Not(), ast.Name(id='content', ctx=ast.Load()), lineno=0, col_offset=0),
ast.Compare(
left=ast.Name(id='tagName', ctx=ast.Load()),
ops=[ast.In()],
comparators=[ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='_void_elements',
ctx=ast.Load()
)]
),
]
),
body=[self._append(ast.Str(u'/>'))],
orelse=[
self._append(ast.Str(u'>')),
ast.If(
test=ast.Name(id='content', ctx=ast.Load()),
body=[self._append(ast.Name(id='content', ctx=ast.Load()))],
orelse=[]
),
self._append(ast.Str(u'</')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
self._append(ast.Str(u'>')),
]
)
],
orelse=[]
)
]
def make_fv(key: str):
return ast.Subscript(value=node.right,
slice=ast.Index(value=ast.Str(s=key)))
def test_mapping_types(self):
# dict
xml = Parser(self.get_xml("dict.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Expr(
ast.Dict(
[ast.Str("name"),
ast.Str("age"),
ast.Str("address")],
[
ast.Str("Batuhan"),
ast.Num(15),
ast.Dict(
[
ast.Str("country"),
ast.Str("city"),
ast.Str("postalcode"),
],
[
ast.Str("Turkey"),
ast.Dict(
[ast.Str("primary"),
ast.Str("secondary")],
[ast.Str("Mersin"),
ast.Str("Konya")],
),
ast.Num(3333),
],
),
],
))
])
ast.fix_missing_locations(mymodule)
mycode = compile(mymodule, "<ast>", "exec")
self.assertEqual(code, mycode)
def to_node(self, obj):
'''
Convert an object to a tuple (funcs, node), in which
func is a list of 0 or more FunctionDefs, and node is a
expression node that creates the object (using the function defs).
'''
if isinstance(obj, (int, long, float)):
return ([], ast.Num(n=obj))
elif isinstance(obj, (str, unicode)):
return ([], ast.Str(s=obj))
elif isinstance(obj, set):
func = []
values = []
for x in obj:
(vfunc, vnode) = self.to_node(x)
func += vfunc
values.append(vnode)
return (func, ast.Set(elts=values))
elif isinstance(obj, dict):
func = []
keys = []
values = []
for key,value in obj.iteritems():
(kfunc, knode) = self.to_node(key)
func += kfunc
(vfunc, vnode) = self.to_node(value)
func += vfunc
keys.append(knode)
values.append(vnode)
return (func, ast.Dict(keys=keys, values=values))
elif obj is None:
return ([], ast.Name('None', ast.Load()))
elif obj is True:
return ([], ast.Name('True', ast.Load()))
elif obj is False:
return ([], ast.Name('False', ast.Load()))
elif hasattr(obj, '__to_node__'):
(func,node) = obj.__to_node__(self)
if isinstance(node, list):
# Multiple statements; build a function around it
(func, node) = self.build_function(func, node)
return (func, node)
else: # Build object from scratch
(objdict_func, objdict_node) = self.to_node(obj.__dict__)
tempvar_ld = ast.Name(id='obj', ctx=ast.Load())
tempvar_st = ast.Name(id='obj', ctx=ast.Store())
body = [
# obj = $cls.__new()
ast.Assign(targets=[tempvar_st], value=ast.Call(
func=ast.Attribute(value=self.class_name(obj), attr='__new__', ctx=ast.Load()),
args=[self.class_name(obj)],
keywords=[],
starargs=None,
kwargs=None
)),
# obj.__dict__ = $objdict
ast.Assign(targets=[ast.Attribute(value=tempvar_ld, attr='__dict__', ctx=ast.Store())],
value=objdict_node),
#
ast.Return(value=tempvar_ld)
]
return self.build_function(objdict_func, body)
def test_sequence_types(self):
# list, tuple, set, frozenset
xml = Parser(self.get_xml("seq.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Expr(
ast.List(
[
ast.Str("Batuhan"),
ast.Num(123),
ast.Num(323.23),
ast.List(
[
ast.Num(235),
ast.Tuple(
[
ast.Str("15"),
ast.Set([
ast.Num(1),
ast.Num(2),
ast.Num(3)
]),
],
ast.Load(),
),
],
ast.Load(),
),
],
ast.Load(),
)),
ast.Expr(
ast.List(
[
ast.Str("Osman"),
ast.Num(321),
ast.Num(3333.333),
ast.List(
[
ast.Num(532),
ast.Tuple(
[
ast.Str("51"),
ast.Set([
ast.Num(3),
ast.Num(2),
ast.Num(1)
]),
],
ast.Load(),
),
],
ast.Load(),
),
],
ast.Load(),
)),
])
ast.fix_missing_locations(mymodule)
mycode = compile(mymodule, "<ast>", "exec")
self.assertEqual(code, mycode)
def gen_assign_name_checker_ast(node):
assert isinstance(node.value, ast.Name)
if debug:
print('old_node: x=y case')
print(dump_tree(node))
rhs_obj_name = node.value.id
new_node = ast.If(
test=ast.Constant(value=True, kind=None),
orelse=[],
body=[
ast.If(
test=ast.Call(
func=ast.Name(id='hasattr', ctx=ast.Load()),
args=[
ast.Name(id=rhs_obj_name, ctx=ast.Load()),
ast.Str(s='__assignpre__'),
],
keywords=[],
starargs=None,
kwargs=None
),
body=[
ast.Assign(
targets=[as_store(lhs_target)],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id=rhs_obj_name, ctx=ast.Load()),
attr='__assignpre__',
ctx=ast.Load()
),
args=[
ast.Str(s=node_name(lhs_target)), # lhs_name
ast.Str(s=rhs_obj_name), # rhs_name
node.value # rhs_value
],
keywords=[],
starargs=None,
kwargs=None
)
) for lhs_target in node.targets],
orelse=[node]
),
ast.If(
test=ast.Call(
func=ast.Name(id='hasattr', ctx=ast.Load()),
args=[
as_load(node.targets[0]),
ast.Str(s='__assignpost__'),
],
keywords=[],
starargs=None,
kwargs=None
),
body=[
ast.Expr(
value=ast.Call(
func=ast.Attribute(
value=as_load(lhs_target),
attr='__assignpost__',
ctx=ast.Load()),
args=[
ast.Str(s=node_name(lhs_target)), # lhs_name
ast.Str(s=node_name(node.value)) # rhs_name
],
keywords=[]
)
) for lhs_target in node.targets],
orelse=[]
)
]
)
if debug:
print('new_node:')
print(dump_tree(new_node))
return new_node
def make_serialize_query_params(self, method: ServiceMethod, node,
module_name: str):
"""Code to serialize optional parameters to the `params` dict passed to
the client post/get call.
This method might also optionally generate a marshmallow schema where it
uses the various traits as base classes.
"""
query_params = {
param.name: param
for param in method.query_params if param.type != "file"
}
# TODO: This should be fixed in the raml specifications since version is
# part of the body and not part of the query parmaters for update calls
if method.type == "update" and "version" in query_params:
del query_params["version"]
# If this method doesn't accept parameters we just exit early with a
# `params = {}` line.
if not query_params:
line = ast.Assign(targets=[ast.Name(id="params")],
value=ast.Dict(keys=[], values=[]))
node.body.append(line)
return
bases = []
for trait in method.traits:
if trait.params:
bases.append(ast.Name(id="traits.%sSchema" % trait.class_name))
# Generate a custom schema if required
if method.extra_params or len(bases) != 1:
if method.type != "action":
schema_name = f"_{method.context_name}{method.type.title()}Schema"
else:
schema_name = f"_{method.context_name}{method.name.title()}Schema"
if not bases:
self.add_import_statement(module_name, "marshmallow", "fields")
self.add_import_statement(module_name, "marshmallow")
bases = [
ast.Name(id="marshmallow.Schema"),
ast.Name(id="RemoveEmptyValuesMixin"),
]
schema_node = ast.ClassDef(name=schema_name,
bases=bases,
keywords=[],
decorator_list=[],
body=[])
# Marshmallow field definitions
schema_methods = []
for param in method.extra_params:
# We skip files since we post the value in the request body
if param.type == "file":
continue
field_node, methods, imports = _create_schema_field(param)
if field_node:
schema_node.body.append(field_node)
schema_methods.extend(methods)
for import_ in imports:
self.add_import_statement(module_name, *import_)
schema_node.body.extend(schema_methods)
if not schema_node.body:
schema_node.body.append(ast.Pass())
self.add_schema(method.context_name, schema_node)
else:
schema_name = bases[0].id
# params = self._serialize_params({}, schema)
input_params = {}
for key, param in query_params.items():
if key.startswith("/"):
key = snakeit(
param.extra_data["(placeholderParam)"]["paramName"])
input_params[key] = snakeit(key)
line = ast.Assign(
targets=[ast.Name(id="params")],
value=ast.Call(
func=ast.Attribute(value=ast.Name(id="self"),
attr="_serialize_params"),
args=[
ast.Dict(
keys=[
ast.Str(s=val, kind="")
for val in input_params.keys()
],
values=[
ast.Name(id=val) for val in input_params.values()
],
),
ast.Name(id=schema_name),
],
keywords=[],
),
)
node.body.append(line)
def ast_subscript(value, attr: str, is_lhs=False):
ctx = (ast.Store if is_lhs else ast.Load)()
return ast.Subscript(value, ast.Str(attr), ctx)
def _generate_init_file(services, modules):
"""Generate the __init__.py file which contains the ServicsMixin for
the client.
This is mostly to automate the addition of new services.
"""
nodes = []
nodes.append(
ast.Import(names=[ast.alias(name="typing", asname=None)], level=0))
nodes.append(
ast.ImportFrom(
module="cached_property",
names=[ast.alias(name="cached_property", asname=None)],
level=0,
))
# Collect all submodules
submodules = {}
for service in services.values():
module_name = snakeit(service.context_name)
service_name = service.context_name + "Service"
info = modules[module_name]
key = ".%s" % info["name"]
submodules[key] = {
"module_name": info["name"],
"class_name": service.context_name + "Service",
"var_name": snakeit(service.context_name),
}
# Add manual generated files (TODO)
submodules[".project"] = {
"module_name": "project",
"class_name": "ProjectService",
"var_name": "project",
}
# Generate TYPE_CHECKING import statements (these will be sorted by isort).
if_node = ast.If(
test=ast.Attribute(value=ast.Name(id="typing"), attr="TYPE_CHECKING"),
body=[],
orelse=[],
)
nodes.append(if_node)
for name, service in submodules.items():
node = ast.ImportFrom(
module=name,
names=[ast.alias(name=service["class_name"], asname=None)],
level=0,
)
if_node.body.append(node)
module_varnames = sorted(submodules.values(),
key=operator.itemgetter("var_name"))
# Return the class + properties
class_node = ast.ClassDef(name="ServicesMixin",
bases=[],
keywords=[],
decorator_list=[],
body=[])
for name, service in submodules.items():
node = ast.FunctionDef(
name=service["module_name"],
args=ast.arguments(
args=[ast.arg(arg="self", annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
),
body=[],
decorator_list=[ast.Name(id="cached_property")],
returns=ast.Str(s=service["class_name"], kind=None),
)
node.body.append(
ast.ImportFrom(
module=name,
names=[ast.alias(name=service["class_name"], asname=None)],
level=0,
))
node.body.append(
ast.Return(
ast.Call(
func=ast.Name(id=service["class_name"]),
args=[ast.Name(id="self")],
keywords=[],
)))
class_node.body.append(node)
nodes.append(class_node)
return ast.Module(body=nodes)
def generate_method_maker() -> t.Tuple[t.List[type], ast.Module]:
def make_match_from_spec(
spec: Spec,
block: BlockLevel = BlockLevel(),
recur=(),
) -> t.List[ast.AST]:
def _make_match_from_spec(spec_, reg_=block):
return make_match_from_spec(spec_, reg_, (*recur, spec_))
if recur.count(spec) is 2 and isinstance(spec, Named):
# avoid recursive expanding
return [
ast_assign(
block.var('ret'),
ast_call(ast_name('make_' + spec.typ.__name__),
[ast_name(block)]))
]
if isinstance(spec, ForwardRef):
raise TypeError
if isinstance(spec, Concrete):
typ = spec.typ
assign_suites = [ast_assign(block.var('ret'), block)]
if typ is object:
return assign_suites
else:
return [
ast.If(ast_isinstance(block, typ), assign_suites, [
ast_raise_type_err(
'expected an instance of ' + repr(typ) +
', got {!r}.', block)
])
]
if isinstance(spec, List):
lst_var = block.var('ret')
append_var = block.var('append')
iter_block = block.let()
method_ast_suites = _make_match_from_spec(spec.elem, iter_block)
return [
ast_assign(lst_var, ast.List([], ast.Load())),
ast_assign(append_var, ast_attr(ast_name(lst_var), 'append')),
ast.For(target=ast_name(iter_block, is_lhs=True),
iter=ast_name(block),
body=[
*method_ast_suites,
ast.Expr(
ast_call(ast_name(append_var),
[ast_name(iter_block.var('ret'))]))
],
orelse=[])
]
if isinstance(spec, Union):
raise NotImplementedError
if isinstance(spec, Dict):
dict_var = block.var('ret')
dict_add_var = dict_var.var('append')
key_var = block.var('key')
value_var = block.var('value')
key_block = block.let().var('key')
value_block = block.let().var('value')
key_match, value_match = _make_match_from_spec(
spec.key,
key_block), _make_match_from_spec(spec.value, value_block)
return [
ast_assign(dict_var, ast.Dict([], [])),
ast_assign(dict_add_var,
ast_attr(ast_name(dict_var), '__setitem__')),
ast.For(
target=ast.Tuple([
ast_name(key_block, is_lhs=True),
ast_name(value_block, is_lhs=True)
], ast.Store()),
iter=ast_call(ast_attr(ast_name(block), 'items'), []),
body=[
*key_match,
ast_assign(key_var, key_block.var('ret')),
*value_match,
ast_assign(value_var, value_block.var('ret')),
ast.Expr(
ast_call(ast_name(dict_add_var),
[ast_name(key_var),
ast_name(value_var)]))
],
orelse=[])
]
if isinstance(spec, Optional):
match_ast_suites = _make_match_from_spec(spec.typ)
return [
ast.If(test=ast_name(block),
body=match_ast_suites,
orelse=[
ast_assign(block.var('ret'), ast.NameConstant(None))
])
]
if isinstance(spec, Named):
named_type = spec.typ
field_block = block.let()
cls_instance_var = block.var('ret')
data_field_getter_var = block.var('append')
def make_match_for_attr(attr: str, field_spec: Spec):
if isinstance(field_spec, ForwardRef):
raise TypeError
else:
extract_suites = _make_match_from_spec(
field_spec, field_block)
return [
ast_assign(
field_block,
ast_call(ast_name(data_field_getter_var),
[ast.Str(attr)])), *extract_suites,
ast.Assign([
ast_attr(
ast_name(cls_instance_var), attr, is_lhs=True)
], ast_name(field_block.var('ret')))
]
_, fields = SchemaMonitor.schemas[named_type.__qualname__]
fields_making = list(map(make_match_for_attr, *zip(*fields)))
return [
ast_assign(cls_instance_var,
ast_call(ast_name('_' + named_type.__name__), [])),
ast_assign(data_field_getter_var,
ast_attr(ast_name(block), '__getitem__')),
*sum(fields_making, [])
]
else:
raise TypeError(spec)
def make_function_ast(ty: type):
nodes = make_match_from_spec(Named(ty))
b = BlockLevel()
ret = ast.FunctionDef(
name='make_' + ty.__name__,
args=ast.arguments(args=[ast.arg(b.to_name(), None)],
vararg=None,
kwonlyargs=[],
kwarg=None,
kw_defaults=[],
defaults=[]),
body=[*nodes, ast.Return(ast_name(b.var('ret')))],
decorator_list=[],
returns=None)
ast.fix_missing_locations(ret)
return ret
types = [a[0] for a in SchemaMonitor.schemas.values()]
fns = list(map(make_function_ast, types))
exports = ast.Return(
ast.Dict(*map(
list,
zip(*[(ast.Str(each.__qualname__),
ast_name('make_' + each.__name__)) for each in types]))))
closure = ast.FunctionDef(name='make',
args=ast.arguments(args=[
ast.arg(each.__name__, None)
for each in types
],
vararg=None,
kwonlyargs=[],
kwarg=None,
kw_defaults=[],
defaults=[]),
body=[*fns, exports],
decorator_list=[],
returns=None)
# optimize name loading to avoid looking up from global context.
collected = CollectLocal()
collected.visit(closure)
local_bindings = collected.names
closure.body = [
*[
ast.Assign([ast_name(name, is_lhs=True)], ast_name(name[1:]))
for name in local_bindings
], *closure.body
]
ast.fix_missing_locations(closure)
# pprint(closure)
mod = ast.Module([closure])
return types, mod
def _create_schema_field(param: ServiceParameter):
"""Generate a field assignment for a marshmallow schema"""
keywords = []
imports = []
methods = []
code_name = snakeit(param.name)
if code_name != param.name:
keywords.append(
ast.keyword(arg="data_key", value=ast.Str(s=param.name,
kind=None)))
if not param.required:
keywords.append(
ast.keyword(arg="required",
value=ast.Constant(value=False, kind=None)))
if param.name.startswith("/"):
placeholder = param.extra_data["(placeholderParam)"]
code_name = snakeit(placeholder["paramName"])
imports.append(("marshmallow", "fields"))
imports.append(("marshmallow", ))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Dict"),
args=[],
keywords=[],
)
# TODO: can break if there is a suffix
key_name = placeholder["template"].replace(
"<%s>" % placeholder["placeholder"], "")
code = ast.parse(
textwrap.dedent(
"""
@marshmallow.post_dump
def _%(target_name)s_post_dump(self, data, **kwrags):
values = data.pop('%(target_name)s')
if not values:
return data
for key, val in values.items():
data[f"%(key_name)s{key}"] = val
return data
@marshmallow.pre_load
def _%(target_name)s_post_load(self, data, **kwrags):
items = {}
for key in list(data.keys()):
if key.startswith("%(key_name)s"):
items[key[%(key_len)d:]] = data[key]
del data[key]
data["%(target_name)s"] = items
return data
""" % {
"target_name": code_name,
"key_name": key_name,
"key_len": len(key_name),
}))
methods.extend(code.body)
elif param.type == "string":
imports.append(("commercetools.helpers", "OptionalList"))
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Name(id="OptionalList"),
args=[
ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"),
attr="String"),
args=[],
keywords=[],
)
],
keywords=keywords,
)
elif param.type == "number":
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Int"),
args=[],
keywords=keywords,
)
elif param.type == "boolean":
keywords.append(
ast.keyword(arg="missing",
value=ast.Constant(value=False, kind=None)))
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Bool"),
args=[],
keywords=keywords,
)
elif param.type == "file":
return None, []
else:
raise NotImplementedError(param)
node = ast.Assign(targets=[ast.Name(id=code_name)],
value=serialize_func,
simple=1)
return node, methods, imports
def _hyperparameter_key_for_param(self, arg):
"""Returns an ``ast.Str`` for a hyperparameter key replacing a legacy mode parameter."""
name = "sagemaker_requirements" if arg == "requirements_file" else arg
return ast.Str(s=name)
def visit_Call(self, node): # pylint: disable=invalid-name
self.generic_visit(node)
# ignore if the function is not 'nni.*'
if type(node.func) is not ast.Attribute:
return node
if type(node.func.value) is not ast.Name:
return node
if node.func.value.id != 'nni':
return node
# ignore if its not a search space function (e.g. `report_final_result`)
func = node.func.attr
if func not in _ss_funcs:
return node
self.last_line = node.lineno
if node.keywords:
# there is a `name` argument
assert len(
node.keywords
) == 1, 'Smart parameter has keyword argument other than "name"'
assert node.keywords[
0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"'
assert type(
node.keywords[0].value
) is ast.Str, 'Smart parameter\'s name must be string literal'
name = node.keywords[0].value.s
specified_name = True
else:
# generate the missing name automatically
name = '__line' + str(str(node.args[-1].lineno))
specified_name = False
node.keywords = list()
if func in ('choice', 'function_choice'):
# we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user
assert len(node.args
) == 1, 'Smart parameter has arguments other than dict'
# check if it is a number or a string and get its value accordingly
args = [
key.n if type(key) is ast.Num else key.s
for key in node.args[0].keys
]
else:
# arguments of other functions must be literal number
assert all(
type(arg) is ast.Num for arg in node.args
), 'Smart parameter\'s arguments must be number literals'
args = [arg.n for arg in node.args]
key = self.module_name + '/' + name + '/' + func
# store key in ast.Call
node.keywords.append(ast.keyword(arg='key', value=ast.Str(s=key)))
if func == 'function_choice':
func = 'choice'
value = {'_type': func, '_value': args}
if specified_name:
# multiple functions with same name must have identical arguments
old = self.search_space.get(key)
assert old is None or old == value, 'Different smart parameters have same name'
else:
# generated name must not duplicate
assert key not in self.search_space, 'Only one smart parameter is allowed in a line'
self.search_space[key] = value
return node