def visit_Assign(self, node): """ Handle `Assign` :param node: Assign :type node: ```Assign``` :returns: `AnnAssign` if `type_annotations` and type found else `Assign` :rtype: ```Union[Assign, AnnAssign]``` """ typ = self._get_ass_typ(node) if self.type_annotations: assert len(node.targets) == 1 return AnnAssign( annotation=typ, value=node.value, type_comment=None, lineno=None, simple=1, target=node.targets[0], expr=None, expr_target=None, expr_annotation=None, ) else: node.type_comment = typ return node
def test_replace_in_ast_with_val_on_non_function(self) -> None: """ Tests that `RewriteAtQuery` can actually replace a node at given location """ parsed_ast = ast_parse(class_str) rewrite_at_query = RewriteAtQuery( search="ConfigClass.dataset_name".split("."), replacement_node=AnnAssign( annotation=Name("int", Load()), simple=1, target=Name("dataset_name", Store()), value=set_value(15), expr=None, expr_target=None, expr_annotation=None, ), ) gen_ast = rewrite_at_query.visit(parsed_ast) self.assertTrue(rewrite_at_query.replaced, True) run_ast_test( self, gen_ast, ast.parse( class_str.replace( 'dataset_name: str = "mnist"', "dataset_name: int = 15" ) ), )
def test_emit_ann_assign(self) -> None: """Tests that AnnAssign is emitted from `emit_ann_assign`""" self.assertIsInstance(class_ast.body[1], AnnAssign) self.assertIsInstance(emit_ann_assign(class_ast.body[1]), AnnAssign) self.assertIsInstance(emit_ann_assign(class_ast.body[1]), AnnAssign) gen_ast = emit_ann_assign( find_in_ast( "C.function_name.dataset_name".split("."), class_with_method_and_body_types_ast, ) ) self.assertIsInstance(gen_ast, AnnAssign) run_ast_test( self, gen_ast, AnnAssign( annotation=Name( "str", Load(), ), simple=1, target=Name("dataset_name", Store()), value=set_value("~/tensorflow_datasets"), expr=None, expr_target=None, expr_annotation=None, ), )
def emit_ann_assign(node): """ Produce an `AnnAssign` from the input :param node: AST node :type node: ```AST``` :return: Something which parses to the form of `a=5` :rtype: ```AnnAssign``` """ if isinstance(node, AnnAssign): return node elif isinstance(node, ast.arg): return AnnAssign( annotation=node.annotation, simple=1, target=Name(node.arg, Store()), value=node.default if hasattr(node, "default") else None, lineno=None, col_offset=None, end_lineno=None, end_col_offset=None, expr=None, expr_target=None, expr_annotation=None, ) else: raise NotImplementedError(type(node).__name__)
def test_annotate_ancestry(self) -> None: """Tests that `annotate_ancestry` properly decorates""" node = Module( body=[ AnnAssign( annotation=Name( "str", Load(), ), simple=1, target=Name("dataset_name", Store()), value=set_value("~/tensorflow_datasets"), expr=None, expr_target=None, expr_annotation=None, ), Assign( annotation=None, simple=1, targets=[Name("epochs", Store())], value=set_value("333"), expr=None, expr_target=None, expr_annotation=None, **maybe_type_comment ), ], stmt=None, ) self.assertFalse(hasattr(node.body[0], "_location")) self.assertFalse(hasattr(node.body[1], "_location")) annotate_ancestry(node) self.assertEqual(node.body[0]._location, ["dataset_name"]) self.assertEqual(node.body[1]._location, ["epochs"])
def visit_ClassDef(self, node): """Process property defaults for Scenic classes.""" if node.name in self.constructors: # Scenic class definition newBody = [] for child in node.body: child = self.visit(child) if isinstance(child, AnnAssign): # default value for property origValue = child.annotation target = child.target # extract any attributes for this property metaAttrs = [] if isinstance(target, Subscript): sl = target.slice if not isinstance(sl, Index): self.parseError(sl, 'malformed attributes for property default') sl = sl.value if isinstance(sl, Name): metaAttrs.append(sl.id) elif isinstance(sl, Tuple): for elt in sl.elts: if not isinstance(elt, Name): self.parseError(elt, 'malformed attributes for property default') metaAttrs.append(elt.id) else: self.parseError(sl, 'malformed attributes for property default') newTarget = Name(target.value.id, Store()) copy_location(newTarget, target) target = newTarget # find dependencies of the default value properties = AttributeFinder.find('self', origValue) # create default value object args = [ Set([Str(prop) for prop in properties]), Set([Str(attr) for attr in metaAttrs]), Lambda(self.self_args, origValue) ] value = Call(Name(createDefault, Load()), args, []) copy_location(value, origValue) newChild = AnnAssign( target=target, annotation=value, value=None, simple=True) child = copy_location(newChild, child) newBody.append(child) node.body = newBody return node else: # ordinary Python class # it's impossible at the moment to define a Python class in a Scenic file, # but we'll leave this check here for future-proofing for base in node.bases: name = None if isinstance(base, Call): name = base.func.id elif isinstance(base, Name): name = base.id if name is not None and name in self.constructors: self.parseError(node, f'Python class {node.name} derives from PRS class {name}') return self.generic_visit(node)
def visit_ClassDef(self, node): """Process property defaults for Scenic classes.""" if node.name in self.constructors: # constructor definition newBody = [] for child in node.body: child = self.visit(child) if isinstance(child, AnnAssign): # default value for property origValue = child.annotation target = child.target # extract any attributes for this property metaAttrs = [] if isinstance(target, Subscript): sl = target.slice if not isinstance(sl, Index): self.parseError(sl, 'malformed attributes for property default') sl = sl.value if isinstance(sl, Name): metaAttrs.append(sl.id) elif isinstance(sl, Tuple): for elt in sl.elts: if not isinstance(elt, Name): self.parseError(elt, 'malformed attributes for property default') metaAttrs.append(elt.id) else: self.parseError(sl, 'malformed attributes for property default') newTarget = Name(target.value.id, Store()) copy_location(newTarget, target) target = newTarget # find dependencies of the default value properties = AttributeFinder.find('self', origValue) # create default value object args = [ Set([Str(prop) for prop in properties]), Set([Str(attr) for attr in metaAttrs]), Lambda(selfArg, origValue) ] value = Call(Name(createDefault, Load()), args, []) copy_location(value, origValue) newChild = AnnAssign( target=target, annotation=value, value=None, simple=True) child = copy_location(newChild, child) newBody.append(child) node.body = newBody return node else: # ordinary Python class # catch some mistakes where 'class' was used instead of 'constructor' for base in node.bases: name = None if isinstance(base, Call): name = base.func.id elif isinstance(base, Name): name = base.id if name is not None and name in self.constructors: self.parseError(node, f'must use "{constructorStatement}" to subclass objects') return self.generic_visit(node)
def make_slide_class(slide: Slide, slide_index: int) -> list: slide_class_name = f"Slide{slide_index}" content_class_name = f"{slide_class_name}Content" klass = [ make_slide_content_class(content_class_name, slide.shapes), ClassDef( name=slide_class_name, bases=[Name(id='SlideData', ctx=Load())], keywords=[], body=[ AnnAssign( target=Name(id='contents', ctx=Store()), annotation=ast_optional_subscript(content_class_name), value=ast_default_field(None), simple=1 ), AnnAssign( target=Name(id='slide_pos', ctx=Store()), annotation=Name(id='int', ctx=Load()), value=Constant(value=slide_index, kind=None), simple=1 ) ], decorator_list=[dataclass_decorator]), ] if slide.has_notes_slide: note = slide.notes_slide.notes_text_frame.text if note_name_regx.match(note): name = note_name_regx.search(note).group('name') alias_slide_class = ClassDef( name=f'{name}Slide', bases=[Name(id=slide_class_name, ctx=Load())], keywords=[], body=[Pass()], decorator_list=[], ) alias_content_class = ClassDef( name=f'{name}Content', bases=[Name(id=content_class_name, ctx=Load())], keywords=[], body=[Pass()], decorator_list=[], ) klass += [alias_slide_class, alias_content_class] return klass
def make_data_field(shape: GraphicFrame): if shape.has_text_frame: value = text_data_value elif shape.has_table: value = table_data_value elif shape.has_chart: value = CHART_DATA_TYPE_MAP.get(shape.chart.chart_type, chart_category_data_value) else: return None return AnnAssign( target=Name(id=name_to_slugify(shape.name), ctx=Store()), annotation=value, value=ast_default_field(None), simple=1, )
def test_param2ast_with_assign(self) -> None: """Check that `param2ast` behaves correctly with a non annotated (typeless) input""" run_ast_test( self, param2ast( ("zion", {"typ": None}), ), gold=AnnAssign( annotation=Name("object", Load()), simple=1, target=Name("zion", Store()), value=set_value(None), expr=None, expr_target=None, expr_annotation=None, ), )
def test_param2ast_with_wrapped_default(self) -> None: """Check that `param2ast` behaves correctly with a wrapped default""" run_ast_test( self, param2ast( ("zion", {"typ": None, "default": set_value(NoneStr)}), ), gold=AnnAssign( annotation=Name("object", Load()), simple=1, target=Name("zion", Store()), value=set_value(None), expr=None, expr_target=None, expr_annotation=None, ), )
def test_param2ast_with_assign_dict(self) -> None: """Check that `param2ast` behaves correctly with dict type""" run_ast_test( self, param2ast( ("menthol", {"typ": "dict"}), ), gold=AnnAssign( annotation=set_slice(Name("dict", Load())), simple=1, target=Name("menthol", Store()), value=Dict(keys=[], values=[], expr=None), expr=None, expr_target=None, expr_annotation=None, ), )
def test_find_in_ast(self) -> None: """Tests that `find_in_ast` successfully finds nodes in AST""" run_ast_test( self, find_in_ast("ConfigClass.dataset_name".split("."), class_ast), AnnAssign( annotation=Name( "str", Load(), ), simple=1, target=Name("dataset_name", Store()), value=set_value("mnist"), expr=None, expr_target=None, expr_annotation=None, ), )
def object_instance_from_res(object_name: str, object_id: str, cls_name: str) -> AnnAssign: try: is_valid_identifier(object_name) except Arcor2Exception as e: raise Arcor2Exception(f"Object name {object_name} invalid. {str(e)}") try: is_valid_type(cls_name) except Arcor2Exception as e: raise Arcor2Exception(f"Class name {cls_name} invalid. {str(e)}") return AnnAssign( target=Name(id=object_name, ctx=Store()), annotation=Name(id=cls_name, ctx=Load()), value=Subscript(value=get_name_attr("res", "objects"), slice=Index(value=Str(s=object_id, kind="")), ctx=Load()), simple=1, )
def test_param2ast_with_bad_default(self) -> None: """Check that `param2ast` behaves correctly with a bad default""" run_ast_test( self, param2ast( ( "stateful_metrics", {"typ": "NoneType", "default": "the `Model`'s metrics"}, ), ), gold=AnnAssign( annotation=Name("NoneType", Load()), simple=1, target=Name("stateful_metrics", Store()), value=set_value("```the `Model`'s metrics```"), expr=None, expr_annotation=None, expr_target=None, ), )
assign_with_type_comment = Assign( targets=[Name("res", Store())], value=BinOp( left=Name("a", Load()), op=Add(), right=Name("b", Load()), ), type_comment=Name("int", Load()), lineno=None, ) ann_assign_with_annotation = AnnAssign( annotation=assign_with_type_comment.type_comment, value=assign_with_type_comment.value, simple=1, target=assign_with_type_comment.targets[0], type_comment=None, expr=None, expr_target=None, expr_annotation=None, lineno=None, ) function_type_annotated = FunctionDef( name="sum", args=arguments( posonlyargs=[], args=[ set_arg(arg="a", annotation=Name("int", Load())), set_arg(arg="b", annotation=Name("int", Load())), ], kwonlyargs=[],
), ClassDef( name='Slide1', bases=[Name(id='SlideData', ctx=Load())], keywords=[], body=[ Assign( targets=[Name(id='slide_pos', ctx=Store())], value=Constant(value=1, kind=None), type_comment=None ), AnnAssign( target=Name(id='contents', ctx=Store()), annotation=Subscript( value=Name(id='Optional', ctx=Load()), slice=Index(value=Name(id='Slide1Content', ctx=Load())), ctx=Load() ), value=None, simple=1) ], decorator_list=[dataclass_decorator]), ClassDef( name='Slide2Content', bases=[], keywords=[], body=[ Assign( targets=[Name(id='title', ctx=Store())], value=text_data_value, type_comment=None
def param2ast(param): """ Converts a param to an AnnAssign :param param: Name, dict with keys: 'typ', 'doc', 'default' :type param: ```Tuple[str, dict]``` :return: AST node for assignment :rtype: ```Union[AnnAssign, Assign]``` """ name, _param = param del param if _param.get("typ") is None and "default" in _param: _param["typ"] = type(_param["default"]).__name__ if "default" in _param and isinstance(_param["default"], (Constant, Str)): _param["default"] = get_value(_param["default"]) if _param["default"] in frozenset((NoneStr, None, "None")): _param["default"] = None if _param["typ"] in frozenset(("Constant", "Str", "NamedConstant")): _param["typ"] = "object" if _param.get("typ") is None: return AnnAssign( annotation=Name("object", Load()), simple=1, target=Name(name, Store()), value=set_value(_param.get("default") or simple_types[_param["typ"]]), expr=None, expr_target=None, expr_annotation=None, ) elif needs_quoting(_param["typ"]): return AnnAssign( annotation=Name(_param["typ"], Load()) if _param["typ"] in simple_types else get_value(ast.parse(_param["typ"]).body[0]), simple=1, target=Name(name, Store()), value=set_value( quote(_param["default"]) if _param.get("default") else simple_types.get(_param["typ"]) ), expr=None, expr_target=None, expr_annotation=None, ) elif _param["typ"] in simple_types: return AnnAssign( annotation=Name(_param["typ"], Load()), simple=1, target=Name(name, Store()), value=set_value(_param.get("default") or simple_types[_param["typ"]]), expr=None, expr_target=None, expr_annotation=None, ) elif _param["typ"] == "dict" or _param["typ"].startswith("*"): return AnnAssign( annotation=set_slice(Name("dict", Load())), simple=1, target=Name(name, Store()), value=Dict(keys=[], values=_param.get("default", []), expr=None), expr=None, expr_target=None, expr_annotation=None, ) else: from doctrans.emitter_utils import ast_parse_fix annotation = ast_parse_fix(_param["typ"]) value = set_value(None) if "default" in _param: if ( isinstance(_param["default"], str) and len(_param["default"]) > 6 and _param["default"].startswith("```") and _param["default"].endswith("```") and _param["default"][3:-3] in frozenset(("None", "(None)")) ): value = set_value(None) else: try: parsed_default = ( set_value(_param["default"]) if ( _param["default"] is None or isinstance(_param["default"], (float, int, str)) ) and not isinstance(_param["default"], str) and not ( isinstance(_param["default"], str) and _param["default"][0] + _param["default"][-1] in frozenset(("()", "[]", "{}")) ) else ast.parse(_param["default"]) ) except SyntaxError: parsed_default = set_value("```{}```".format(_param["default"])) value = ( parsed_default.body[0].value if hasattr(parsed_default, "body") else parsed_default if "default" in _param else set_value(None) ) return AnnAssign( annotation=annotation, simple=1, target=Name(name, Store()), value=value, expr=None, expr_target=None, expr_annotation=None, )
def expr_stmt(self, s: List[Tree]): name = s[0] augment: Union[None, Tree] if len(s) == 2: augment = None value = s[1] else: augment, value = s[1:] annotation: Union[None, Tree] = None if isinstance(value, Tree): if value.data == 'annassign': annotation = value.children[0] value = value.children[1] temp = [] for n in name: if isinstance(n, Name): n = self.StoreName(id=n.id) temp.append(n) name = temp.copy() if annotation is not None or augment is not None: if isinstance(name, list): name = name[0] if augment is not None: func: Union[None, Add, Sub, Mult, MatMult, Div, FloorDiv, Mod, Pow, BitAnd, BitOr, BitXor] = None aug: Token = augment.children[0] func = { '+=': Add, '-=': Sub, '*=': Mult, '@=': MatMult, '/=': Div, '//=': FloorDiv, '%=': Mod, '**=': Pow, '&=': BitAnd, '|=': BitOr, '^=': BitXor, '<<=': LShift, '>>=': RShift, }.get(aug.value, None) if func is not None: return AugAssign(name, func(), value) if annotation is not None: if isinstance(annotation, list): annotation = annotation[0] if isinstance(annotation, Name): annotation = self.LoadName(annotation.id) return AnnAssign(target=name, value=value, simple=1, annotation=annotation) return Assign(targets=name, value=value)
"\n Acquire from the official tensorflow_datasets model zoo," " or the ophthalmology focussed ml-prepare library\n\n " ':cvar dataset_name: name of dataset. Defaults to "mnist"\n ' ':cvar tfds_dir: directory to look for models in. Defaults to "~/tensorflow_datasets"\n ' ':cvar K: backend engine, e.g., `np` or `tf`. Defaults to "np"\n ' ":cvar as_numpy: Convert to numpy ndarrays. Defaults to None\n " ":cvar data_loader_kwargs: pass this as arguments to data_loader function\n " ":cvar return_type: Train and tests dataset splits. Defaults to (np.empty(0), np.empty(0))", ) ), AnnAssign( annotation=Name( "str", Load(), ), simple=1, target=Name("dataset_name", Store()), value=set_value("mnist"), expr=None, expr_annotation=None, expr_target=None, ), AnnAssign( annotation=Name( "str", Load(), ), simple=1, target=Name("tfds_dir", Store()), value=set_value( "~/tensorflow_datasets", ),
def param2ast(param): """ Converts a param to an AnnAssign :param param: Name, dict with keys: 'typ', 'doc', 'default' :type param: ```Tuple[str, dict]``` :returns: AST node for assignment :rtype: ```Union[AnnAssign, Assign]``` """ name, _param = param del param if _param.get("typ") is None and "default" in _param and "[" not in _param: _param["typ"] = type(_param["default"]).__name__ if "default" in _param: if isinstance(_param["default"], (Constant, Str)): _param["default"] = get_value(_param["default"]) if _param["default"] in none_types: _param["default"] = None if _param["typ"] in frozenset(("Constant", "Str", "NamedConstant")): _param["typ"] = "object" elif _param["default"] == NoneStr: _param["default"] = None if _param.get("typ") is None: return AnnAssign( annotation=Name("object", Load()), simple=1, target=Name(name, Store()), value=set_value(_param.get("default")), expr=None, expr_target=None, expr_annotation=None, ) # return Assign( # annotation=None, # simple=1, # targets=[Name(name, Store())], # value=set_value(_param.get("default")), # expr=None, # expr_target=None, # expr_annotation=None, # **maybe_type_comment # ) elif needs_quoting(_param["typ"]): return AnnAssign( annotation=Name(_param["typ"], Load()) if _param["typ"] in simple_types else get_value(ast.parse(_param["typ"]).body[0]), simple=1, target=Name(name, Store()), value=set_value( quote(_param["default"]) if _param.get("default") else simple_types.get(_param["typ"]) ), expr=None, expr_target=None, expr_annotation=None, ) elif _param["typ"] in simple_types: return AnnAssign( annotation=Name(_param["typ"], Load()), simple=1, target=Name(name, Store()), value=set_value( None if _param.get("default") == NoneStr else (_param.get("default") or simple_types[_param["typ"]]) ), expr=None, expr_target=None, expr_annotation=None, ) elif _param["typ"] == "dict" or _param["typ"].startswith("*"): return AnnAssign( annotation=set_slice(Name("dict", Load())), simple=1, target=Name(name, Store()), value=Dict(keys=[], values=_param.get("default", []), expr=None), expr=None, expr_target=None, expr_annotation=None, ) else: return _generic_param2ast((name, _param))
def _generic_param2ast(param): """ Internal function to turn a param into an `AnnAssign`. Expected to be used only inside `param2ast`. :param param: Name, dict with keys: 'typ', 'doc', 'default' :type param: ```Tuple[str, dict]``` :returns: AST node for assignment :rtype: ```AnnAssign``` """ name, _param = param del param from doctrans.emitter_utils import ast_parse_fix annotation = ast_parse_fix(_param["typ"]) value = set_value(None) if "default" in _param: if not code_quoted(_param["default"]) or _param["default"][ 3:-3 ] not in frozenset(("None", "(None)")): try: parsed_default = ( set_value(_param["default"]) if ( _param["default"] is None or isinstance(_param["default"], (float, int, str)) ) and not isinstance(_param["default"], str) and not ( isinstance(_param["default"], str) and _param["default"][0] + _param["default"][-1] in frozenset(("()", "[]", "{}")) ) else ast.parse(_param["default"]) ) except SyntaxError: parsed_default = set_value( _param["default"] if code_quoted(_param["default"]) else "```{}```".format(_param["default"]) ) value = ( parsed_default.body[0].value if hasattr(parsed_default, "body") else parsed_default if "default" in _param else set_value(None) ) # else: # value = set_value(None) return AnnAssign( annotation=annotation, simple=1, target=Name(name, Store()), value=value, expr=None, expr_target=None, expr_annotation=None, )