def post_install_clean_up(): """Do some post-processing on the installation to clean things like property inheritance""" # Find all the intermediate properties that are using a base class and create an appropriate # inherited class that matches the inheriance of its parent class # Example: say someone registers "graph/colors/red" # We end up with _BaseGraphColors, where _BaseGraph has a property 'colors' which points to it # Now, regular old Graph(_BaseGraph) did get any commands installed at colors specifically, so it inherits property 'color' which points to _BaseGraphColors # This makes things awkward because the Graph class installation doesn't know about the property color, since it wasn't installed there. # It becomes much more straightforward in Graph gets its own proeprty 'colors' (overriding inheritance) which points to a class GraphColors # So in this method, we look for this situation, now that installation is complete. installable_classes = metaprog.get_installable_classes() for cls in installable_classes: installation = metaprog.get_installation(cls) if installation: for name in dir(cls): member = getattr(cls, name) if metaprog.is_intermediate_property(member): if name not in cls.__dict__: # is inherited... path =metaprog.InstallPath(installation.install_path.entity_type + '/' + name) new_class = metaprog._create_class(path) # create new, appropriate class to avoid inheritance installation.add_property(cls, name, new_class) metaprog.get_installation(new_class).keep_me = True # mark its installation so it will survive the next for loop # Some added properties may access intermediate classes which did not end up # receiving any commands. They are empty and should not appear in the API. We # will take a moment to delete them. This approach seemed much better than writing # special logic to calculate fancy inheritance and awkwardly insert classes into # the hierarchy on-demand. for name, intermediate_cls in installation.intermediates.items(): intermediate_installation = metaprog.get_installation(intermediate_cls) if not intermediate_installation.commands and not hasattr(intermediate_installation, "keep_me"): delattr(cls, name) del installation.intermediates[name]
def post_install_clean_up(): """Do some post-processing on the installation to clean things like property inheritance""" # Find all the intermediate properties that are using a base class and create an appropriate # inherited class that matches the inheriance of its parent class # Example: say someone registers "graph/colors/red" # We end up with _BaseGraphColors, where _BaseGraph has a property 'colors' which points to it # Then we see "graph:titan/colors/blue" # We get TitanGraphColors(_BaseGraphColors) and TitaGraph has a property 'colors' which points to it # Now, regular old Graph(_BaseGraph) did get any commands installed at colors specifically, so it inherits property 'color' which points to _BaseGraphColors # This makes things awkward because the Graph class installation doesn't know about the property color, since it wasn't installed there. # It becomes much more straightforward in Graph gets its own proeprty 'colors' (overriding inheritance) which points to a class GraphColors # So in this method, we look for this situation, now that installation is complete. installable_classes = metaprog.get_installable_classes() for cls in installable_classes: installation = metaprog.get_installation(cls) if installation: for name in dir(cls): member = getattr(cls, name) if metaprog.is_intermediate_property(member): if name not in cls.__dict__: # is inherited... path =metaprog.InstallPath(installation.install_path.entity_type + '/' + name) new_class = metaprog._create_class(path) # create new, appropriate class to avoid inheritance installation.add_property(cls, name, new_class) metaprog.get_installation(new_class).keep_me = True # mark its installation so it will survive the next for loop # Some added properties may access intermediate classes which did not end up # receiving any commands. They are empty and should not appear in the API. We # will take a moment to delete them. This approach seemed much better than writing # special logic to calculate fancy inheritance and awkwardly insert classes into # the hierarchy on-demand. for name, intermediate_cls in installation.intermediates.items(): intermediate_installation = metaprog.get_installation(intermediate_cls) if not intermediate_installation.commands and not hasattr(intermediate_installation, "keep_me"): delattr(cls, name) del installation.intermediates[name]
def install_command_def(cls, command_def, execute_command_function): """Adds command dynamically to the loadable_class""" if command_def.full_name not in muted_commands: if is_command_name_installable(cls, command_def.name): check_loadable_class(cls, command_def) function = metaprog.create_function(cls, command_def, execute_command_function) function._is_api = True if command_def.is_constructor: cls.__init__ = function cls.__repr__ = metaprog.get_repr_function(command_def) else: setattr(cls, command_def.name, function) metaprog.get_installation(cls).commands.append(command_def) #print "%s <-- %s" % (cls.__name__, command_def.full_name) logger.debug("Installed api function %s to class %s", command_def.name, cls)
def install_command_def(cls, command_def, execute_command_function): """Adds command dynamically to the loadable_class""" if command_def.full_name not in muted_commands: if is_command_name_installable(cls, command_def.name): check_loadable_class(cls, command_def) function = metaprog.create_function(cls, command_def, execute_command_function) function._is_api = True if command_def.is_constructor: cls.__init__ = function cls.__repr__ = metaprog.get_repr_function(command_def) else: setattr(cls, command_def.name, function) metaprog.get_installation(cls).commands.append(command_def) # print "%s <-- %s" % (cls.__name__, command_def.full_name) logger.debug("Installed api function %s to class %s", command_def.name, cls)
def install_client_commands(): cleared_classes = set() for class_name, command_def in client_commands: # what to do with global methods??? # validation cls = None if class_name: try: cls = [c for c in api_globals if inspect.isclass(c) and c.__name__ == class_name][0] except IndexError: #raise RuntimeError("Internal Error: @api decoration cannot resolve class name %s for function %s" % (class_name, command_def.name)) pass if cls: cls_with_installation = metaprog.get_class_from_store(metaprog.InstallPath(command_def.entity_type)) if cls_with_installation is not cls: raise RuntimeError("Internal Error: @api decoration resolved with mismatched classes for %s" % class_name) if cls not in cleared_classes: clear_clientside_api_stubs(cls) cleared_classes.add(cls) installation = metaprog.get_installation(cls_with_installation) if command_def.entity_type != installation.install_path.full: raise RuntimeError("Internal Error: @api decoration resulted in different install paths '%s' and '%s' for function %s in class %s" % (command_def.entity_type, installation.install_path, command_def.name, class_name)) if command_def.full_name not in muted_commands: installation.commands.append(command_def) setattr(cls, command_def.name, command_def.client_member) logger.debug("Installed client-side api function %s to class %s", command_def.name, cls) elif command_def.client_member not in api_globals: # global function api_globals.add(command_def.client_member)
def get_doc_stub_modules_text(class_to_member_text_dict, import_return_types): """creates spa text for two different modules, returning the content in a tuple""" # The first module contains dependencies for the entity classes that are 'hard-coded' in the python API, # like Frame, Graph... They need things like _DocStubsFrame, or GraphMl to be defined first. # The second module contains entity classes that are created by meta-programming, like LdaModel, *Model, # which may depend on the 'hard-coded' python API. The second modules also contains any global methods, # like get_frame_names, which depend on objects like Frame being already defined. module1_lines = [get_file_header_text()] module2_lines = [] module2_all = [] # holds the names which should be in module2's __all__ for import * classes = sorted([(k, v) for k, v in class_to_member_text_dict.items()], key=lambda kvp: kvp[0].__name__) for cls, members_info in classes: logger.info("Processing %s for doc stubs", cls) names, text = zip(*members_info) installation = get_installation(cls, None) if installation: class_name, baseclass_name = installation.install_path.get_class_and_baseclass_names() if class_name not in exclude_classes_list: if class_name != cls.__name__: raise RuntimeError( "Internal Error: class name mismatch generating docstubs (%s != %s)" % (class_name, cls.__name__) ) if installation.host_class_was_created and installation.install_path.is_entity: lines = module2_lines module2_all.append(class_name) else: if not installation.host_class_was_created: class_name = get_doc_stub_class_name(class_name) lines = module1_lines lines.append( get_doc_stubs_class_text( class_name, "object", # no inheritance for docstubs, just define all explicitly "Auto-generated to contain doc stubs for static program analysis", indent("\n\n".join(text)), ) ) elif cls.__name__ == "trustedanalytics": module2_lines.extend(list(text)) module2_all.extend(list(names)) module2_lines.insert(0, '\n__all__ = ["%s"]' % '", "'.join(module2_all)) # Need to import any return type to enable SPA, like for get_frame, we need Frame for t in import_return_types: if test_import(t): module2_lines.insert(0, "from trustedanalytics import %s" % t) module2_lines.insert(0, get_file_header_text()) # remove doc_stub from namespace module1_lines.append(get_file_footer_text()) module2_lines.append("\ndel doc_stub") return "\n".join(module1_lines), "\n".join(module2_lines)
def get_class_rst(cls): """ Gets rst text to describe a class Only lists members in summary tables with hyperlinks to other rst docs, with the exception of __init__ which is described in class rst text, below the summary tables. """ sorted_members = get_member_rst_list(cls) # sort defs for both summary table and attribute tables toctree = indent("\n".join([m.rst_name for m in sorted_members])) starter = """ .. toctree:: :hidden: {toctree} .. class:: {name} {rst_doc}""".format(toctree=toctree, name=get_type_name(cls), rst_doc=indent(cls.__doc__)) lines = [starter] lines.append(indent(get_attribute_summary_table(sorted_members), 4)) lines.append(indent(get_method_summary_table(sorted_members), 4)) installation = get_installation(cls) init_commands = [c for c in installation.commands if c.is_constructor] if init_commands: lines.append("\n.. _%s:\n" % get_cls_init_rst_label(cls)) # add rst label for internal reference lines.append(get_command_def_rst(init_commands[0])) return "\n".join(lines)
def get_doc_stub_modules_text(class_to_member_text_dict, import_return_types): """creates spa text for two different modules, returning the content in a tuple""" # The first module contains dependencies for the entity classes that are 'hard-coded' in the python API, # like Frame, Graph... They need things like _DocStubsFrame, or GraphMl to be defined first. # The second module contains entity classes that are created by meta-programming, like LdaModel, *Model, # which may depend on the 'hard-coded' python API. The second modules also contains any global methods, # like get_frame_names, which depend on objects like Frame being already defined. module1_lines = [get_file_header_text()] module2_lines = [] module2_all = [ ] # holds the names which should be in module2's __all__ for import * classes = sorted([(k, v) for k, v in class_to_member_text_dict.items()], key=lambda kvp: kvp[0].__name__) for cls, members_info in classes: logger.info("Processing %s for doc stubs", cls) names, text = zip(*members_info) installation = get_installation(cls, None) if installation: class_name, baseclass_name = installation.install_path.get_class_and_baseclass_names( ) if class_name not in exclude_classes_list: if class_name != cls.__name__: raise RuntimeError( "Internal Error: class name mismatch generating docstubs (%s != %s)" % (class_name, cls.__name__)) if installation.host_class_was_created and installation.install_path.is_entity: lines = module2_lines module2_all.append(class_name) else: if not installation.host_class_was_created: class_name = get_doc_stub_class_name(class_name) lines = module1_lines lines.append( get_doc_stubs_class_text( class_name, "object", # no inheritance for docstubs, just define all explicitly "Auto-generated to contain doc stubs for static program analysis", indent("\n\n".join(text)))) elif cls.__name__ == "trustedanalytics": module2_lines.extend(list(text)) module2_all.extend(list(names)) module2_lines.insert(0, '\n__all__ = ["%s"]' % '", "'.join(module2_all)) # Need to import any return type to enable SPA, like for get_frame, we need Frame for t in import_return_types: if test_import(t): module2_lines.insert(0, "from trustedanalytics import %s" % t) module2_lines.insert(0, get_file_header_text()) # remove doc_stub from namespace module1_lines.append(get_file_footer_text()) module2_lines.append("\ndel doc_stub") return '\n'.join(module1_lines), '\n'.join(module2_lines)
def write_py_rst_cls_file(root_path, cls): installation = get_installation(cls) entity_collection_name = installation.install_path.entity_collection_name header = get_rst_cls_file_header(entity_collection_name, class_name=get_type_name(cls)) content = get_class_rst(cls) folder = get_rst_folder_path(root_path, cls) file_path = os.path.join(folder, "index.rst") write_text_to_file(file_path, [header, content])
def get_entity_collection_name_and_install_path(obj, attr): installation = get_installation(obj, None) if installation: entity_collection_name = installation.install_path.entity_collection_name install_path = installation.install_path.full.replace(':', '-') elif attr and has_entity_collection(attr): entity_collection_name = get_entity_collection(attr) install_path = '' else: raise RuntimeError("Unable to determine the entity collection for method %s" % attr) return entity_collection_name, install_path
def get_entity_collection_name_and_install_path(obj, attr): installation = get_installation(obj, None) if installation: entity_collection_name = installation.install_path.entity_collection_name install_path = installation.install_path.full.replace(':', '-') elif attr and has_entity_collection(attr): entity_collection_name = get_entity_collection(attr) install_path = '' else: raise RuntimeError( "Unable to determine the entity collection for method %s" % attr) return entity_collection_name, install_path
def get_entity_collection_name_and_install_path(obj, attr): installation = get_installation(obj, None) if installation: entity_collection_name = installation.install_path.entity_collection_name install_path = installation.install_path.full.replace(':', '-') elif attr and has_entity_collection(attr): entity_collection_name = get_entity_collection(attr) install_path = '' else: # todo: fix me entity_collection_name = "frames" # hack to put global items in frames, until we get a global area in the docs install_path = '' # raise RuntimeError("Unable to determine the entity collection for method %s" % attr) return entity_collection_name, install_path
def check_loadable_class(cls, command_def): error = None installation = metaprog.get_installation(cls) if not installation: error = "class %s is not prepared for command installation" % cls elif command_def.is_constructor and not installation.install_path.is_entity: raise RuntimeError("API load error: special method '%s' may only be defined on entity classes, not on entity " "member classes (i.e. only one slash allowed)." % command_def.full_name) elif command_def.entity_type != installation.install_path.entity_type: error = "%s is not the class's accepted command entity_type: %s" \ % (command_def.entity_type, installation.install_path.entity_type) if error: raise ValueError("API load error: Class %s cannot load command_def '%s'.\n%s" % (cls.__name__, command_def.full_name, error))
def get_index_ref(cls): installation = get_installation(cls, None) nested_level = len( installation.install_path._intermediate_names) if installation else 0 return ("../" * nested_level) + 'index'
def get_index_ref(cls): installation = get_installation(cls, None) nested_level = len(installation.install_path._intermediate_names) if installation else 0 return ("../" * nested_level) + 'index'