def post_install_clean_up():
"""Do some post-processing on the installation to clean things like property inheritance"""
# Find all the intermediate properties that are using a base class and create an appropriate
# inherited class that matches the inheriance of its parent class
# Example: say someone registers "graph/colors/red"
# We end up with _BaseGraphColors, where _BaseGraph has a property 'colors' which points to it
# Now, regular old Graph(_BaseGraph) did get any commands installed at colors specifically, so it inherits property 'color' which points to _BaseGraphColors
# This makes things awkward because the Graph class installation doesn't know about the property color, since it wasn't installed there.
# It becomes much more straightforward in Graph gets its own proeprty 'colors' (overriding inheritance) which points to a class GraphColors
# So in this method, we look for this situation, now that installation is complete.
installable_classes = metaprog.get_installable_classes()
for cls in installable_classes:
installation = metaprog.get_installation(cls)
if installation:
for name in dir(cls):
member = getattr(cls, name)
if metaprog.is_intermediate_property(member):
if name not in cls.__dict__: # is inherited...
path =metaprog.InstallPath(installation.install_path.entity_type + '/' + name)
new_class = metaprog._create_class(path) # create new, appropriate class to avoid inheritance
installation.add_property(cls, name, new_class)
metaprog.get_installation(new_class).keep_me = True # mark its installation so it will survive the next for loop
# Some added properties may access intermediate classes which did not end up
# receiving any commands. They are empty and should not appear in the API. We
# will take a moment to delete them. This approach seemed much better than writing
# special logic to calculate fancy inheritance and awkwardly insert classes into
# the hierarchy on-demand.
for name, intermediate_cls in installation.intermediates.items():
intermediate_installation = metaprog.get_installation(intermediate_cls)
if not intermediate_installation.commands and not hasattr(intermediate_installation, "keep_me"):
delattr(cls, name)
del installation.intermediates[name]
def post_install_clean_up():
"""Do some post-processing on the installation to clean things like property inheritance"""
# Find all the intermediate properties that are using a base class and create an appropriate
# inherited class that matches the inheriance of its parent class
# Example: say someone registers "graph/colors/red"
# We end up with _BaseGraphColors, where _BaseGraph has a property 'colors' which points to it
# Then we see "graph:titan/colors/blue"
# We get TitanGraphColors(_BaseGraphColors) and TitaGraph has a property 'colors' which points to it
# Now, regular old Graph(_BaseGraph) did get any commands installed at colors specifically, so it inherits property 'color' which points to _BaseGraphColors
# This makes things awkward because the Graph class installation doesn't know about the property color, since it wasn't installed there.
# It becomes much more straightforward in Graph gets its own proeprty 'colors' (overriding inheritance) which points to a class GraphColors
# So in this method, we look for this situation, now that installation is complete.
installable_classes = metaprog.get_installable_classes()
for cls in installable_classes:
installation = metaprog.get_installation(cls)
if installation:
for name in dir(cls):
member = getattr(cls, name)
if metaprog.is_intermediate_property(member):
if name not in cls.__dict__: # is inherited...
path =metaprog.InstallPath(installation.install_path.entity_type + '/' + name)
new_class = metaprog._create_class(path) # create new, appropriate class to avoid inheritance
installation.add_property(cls, name, new_class)
metaprog.get_installation(new_class).keep_me = True # mark its installation so it will survive the next for loop
# Some added properties may access intermediate classes which did not end up
# receiving any commands. They are empty and should not appear in the API. We
# will take a moment to delete them. This approach seemed much better than writing
# special logic to calculate fancy inheritance and awkwardly insert classes into
# the hierarchy on-demand.
for name, intermediate_cls in installation.intermediates.items():
intermediate_installation = metaprog.get_installation(intermediate_cls)
if not intermediate_installation.commands and not hasattr(intermediate_installation, "keep_me"):
delattr(cls, name)
del installation.intermediates[name]
def install_command_def(cls, command_def, execute_command_function):
"""Adds command dynamically to the loadable_class"""
if command_def.full_name not in muted_commands:
if is_command_name_installable(cls, command_def.name):
check_loadable_class(cls, command_def)
function = metaprog.create_function(cls, command_def, execute_command_function)
function._is_api = True
if command_def.is_constructor:
cls.__init__ = function
cls.__repr__ = metaprog.get_repr_function(command_def)
else:
setattr(cls, command_def.name, function)
metaprog.get_installation(cls).commands.append(command_def)
#print "%s <-- %s" % (cls.__name__, command_def.full_name)
logger.debug("Installed api function %s to class %s", command_def.name, cls)
def install_command_def(cls, command_def, execute_command_function):
"""Adds command dynamically to the loadable_class"""
if command_def.full_name not in muted_commands:
if is_command_name_installable(cls, command_def.name):
check_loadable_class(cls, command_def)
function = metaprog.create_function(cls, command_def, execute_command_function)
function._is_api = True
if command_def.is_constructor:
cls.__init__ = function
cls.__repr__ = metaprog.get_repr_function(command_def)
else:
setattr(cls, command_def.name, function)
metaprog.get_installation(cls).commands.append(command_def)
# print "%s <-- %s" % (cls.__name__, command_def.full_name)
logger.debug("Installed api function %s to class %s", command_def.name, cls)
def install_client_commands():
cleared_classes = set()
for class_name, command_def in client_commands:
# what to do with global methods???
# validation
cls = None
if class_name:
try:
cls = [c for c in api_globals if inspect.isclass(c) and c.__name__ == class_name][0]
except IndexError:
#raise RuntimeError("Internal Error: @api decoration cannot resolve class name %s for function %s" % (class_name, command_def.name))
pass
if cls:
cls_with_installation = metaprog.get_class_from_store(metaprog.InstallPath(command_def.entity_type))
if cls_with_installation is not cls:
raise RuntimeError("Internal Error: @api decoration resolved with mismatched classes for %s" % class_name)
if cls not in cleared_classes:
clear_clientside_api_stubs(cls)
cleared_classes.add(cls)
installation = metaprog.get_installation(cls_with_installation)
if command_def.entity_type != installation.install_path.full:
raise RuntimeError("Internal Error: @api decoration resulted in different install paths '%s' and '%s' for function %s in class %s"
% (command_def.entity_type, installation.install_path, command_def.name, class_name))
if command_def.full_name not in muted_commands:
installation.commands.append(command_def)
setattr(cls, command_def.name, command_def.client_member)
logger.debug("Installed client-side api function %s to class %s", command_def.name, cls)
elif command_def.client_member not in api_globals:
# global function
api_globals.add(command_def.client_member)
def get_doc_stub_modules_text(class_to_member_text_dict, import_return_types):
"""creates spa text for two different modules, returning the content in a tuple"""
# The first module contains dependencies for the entity classes that are 'hard-coded' in the python API,
# like Frame, Graph... They need things like _DocStubsFrame, or GraphMl to be defined first.
# The second module contains entity classes that are created by meta-programming, like LdaModel, *Model,
# which may depend on the 'hard-coded' python API. The second modules also contains any global methods,
# like get_frame_names, which depend on objects like Frame being already defined.
module1_lines = [get_file_header_text()]
module2_lines = []
module2_all = [] # holds the names which should be in module2's __all__ for import *
classes = sorted([(k, v) for k, v in class_to_member_text_dict.items()], key=lambda kvp: kvp[0].__name__)
for cls, members_info in classes:
logger.info("Processing %s for doc stubs", cls)
names, text = zip(*members_info)
installation = get_installation(cls, None)
if installation:
class_name, baseclass_name = installation.install_path.get_class_and_baseclass_names()
if class_name not in exclude_classes_list:
if class_name != cls.__name__:
raise RuntimeError(
"Internal Error: class name mismatch generating docstubs (%s != %s)"
% (class_name, cls.__name__)
)
if installation.host_class_was_created and installation.install_path.is_entity:
lines = module2_lines
module2_all.append(class_name)
else:
if not installation.host_class_was_created:
class_name = get_doc_stub_class_name(class_name)
lines = module1_lines
lines.append(
get_doc_stubs_class_text(
class_name,
"object", # no inheritance for docstubs, just define all explicitly
"Auto-generated to contain doc stubs for static program analysis",
indent("\n\n".join(text)),
)
)
elif cls.__name__ == "trustedanalytics":
module2_lines.extend(list(text))
module2_all.extend(list(names))
module2_lines.insert(0, '\n__all__ = ["%s"]' % '", "'.join(module2_all))
# Need to import any return type to enable SPA, like for get_frame, we need Frame
for t in import_return_types:
if test_import(t):
module2_lines.insert(0, "from trustedanalytics import %s" % t)
module2_lines.insert(0, get_file_header_text())
# remove doc_stub from namespace
module1_lines.append(get_file_footer_text())
module2_lines.append("\ndel doc_stub")
return "\n".join(module1_lines), "\n".join(module2_lines)
def get_class_rst(cls):
"""
Gets rst text to describe a class
Only lists members in summary tables with hyperlinks to other rst docs, with the exception of __init__
which is described in class rst text, below the summary tables.
"""
sorted_members = get_member_rst_list(cls) # sort defs for both summary table and attribute tables
toctree = indent("\n".join([m.rst_name for m in sorted_members]))
starter = """
.. toctree::
:hidden:
{toctree}
.. class:: {name}
{rst_doc}""".format(toctree=toctree, name=get_type_name(cls), rst_doc=indent(cls.__doc__))
lines = [starter]
lines.append(indent(get_attribute_summary_table(sorted_members), 4))
lines.append(indent(get_method_summary_table(sorted_members), 4))
installation = get_installation(cls)
init_commands = [c for c in installation.commands if c.is_constructor]
if init_commands:
lines.append("\n.. _%s:\n" % get_cls_init_rst_label(cls)) # add rst label for internal reference
lines.append(get_command_def_rst(init_commands[0]))
return "\n".join(lines)
def get_doc_stub_modules_text(class_to_member_text_dict, import_return_types):
"""creates spa text for two different modules, returning the content in a tuple"""
# The first module contains dependencies for the entity classes that are 'hard-coded' in the python API,
# like Frame, Graph... They need things like _DocStubsFrame, or GraphMl to be defined first.
# The second module contains entity classes that are created by meta-programming, like LdaModel, *Model,
# which may depend on the 'hard-coded' python API. The second modules also contains any global methods,
# like get_frame_names, which depend on objects like Frame being already defined.
module1_lines = [get_file_header_text()]
module2_lines = []
module2_all = [
] # holds the names which should be in module2's __all__ for import *
classes = sorted([(k, v) for k, v in class_to_member_text_dict.items()],
key=lambda kvp: kvp[0].__name__)
for cls, members_info in classes:
logger.info("Processing %s for doc stubs", cls)
names, text = zip(*members_info)
installation = get_installation(cls, None)
if installation:
class_name, baseclass_name = installation.install_path.get_class_and_baseclass_names(
)
if class_name not in exclude_classes_list:
if class_name != cls.__name__:
raise RuntimeError(
"Internal Error: class name mismatch generating docstubs (%s != %s)"
% (class_name, cls.__name__))
if installation.host_class_was_created and installation.install_path.is_entity:
lines = module2_lines
module2_all.append(class_name)
else:
if not installation.host_class_was_created:
class_name = get_doc_stub_class_name(class_name)
lines = module1_lines
lines.append(
get_doc_stubs_class_text(
class_name,
"object", # no inheritance for docstubs, just define all explicitly
"Auto-generated to contain doc stubs for static program analysis",
indent("\n\n".join(text))))
elif cls.__name__ == "trustedanalytics":
module2_lines.extend(list(text))
module2_all.extend(list(names))
module2_lines.insert(0, '\n__all__ = ["%s"]' % '", "'.join(module2_all))
# Need to import any return type to enable SPA, like for get_frame, we need Frame
for t in import_return_types:
if test_import(t):
module2_lines.insert(0, "from trustedanalytics import %s" % t)
module2_lines.insert(0, get_file_header_text())
# remove doc_stub from namespace
module1_lines.append(get_file_footer_text())
module2_lines.append("\ndel doc_stub")
return '\n'.join(module1_lines), '\n'.join(module2_lines)
def install_client_commands():
cleared_classes = set()
for class_name, command_def in client_commands:
# what to do with global methods???
# validation
cls = None
if class_name:
try:
cls = [c for c in api_globals if inspect.isclass(c) and c.__name__ == class_name][0]
except IndexError:
#raise RuntimeError("Internal Error: @api decoration cannot resolve class name %s for function %s" % (class_name, command_def.name))
pass
if cls:
cls_with_installation = metaprog.get_class_from_store(metaprog.InstallPath(command_def.entity_type))
if cls_with_installation is not cls:
raise RuntimeError("Internal Error: @api decoration resolved with mismatched classes for %s" % class_name)
if cls not in cleared_classes:
clear_clientside_api_stubs(cls)
cleared_classes.add(cls)
installation = metaprog.get_installation(cls_with_installation)
if command_def.entity_type != installation.install_path.full:
raise RuntimeError("Internal Error: @api decoration resulted in different install paths '%s' and '%s' for function %s in class %s"
% (command_def.entity_type, installation.install_path, command_def.name, class_name))
if command_def.full_name not in muted_commands:
installation.commands.append(command_def)
setattr(cls, command_def.name, command_def.client_member)
logger.debug("Installed client-side api function %s to class %s", command_def.name, cls)
elif command_def.client_member not in api_globals:
# global function
api_globals.add(command_def.client_member)
def get_class_rst(cls):
"""
Gets rst text to describe a class
Only lists members in summary tables with hyperlinks to other rst docs, with the exception of __init__
which is described in class rst text, below the summary tables.
"""
sorted_members = get_member_rst_list(cls) # sort defs for both summary table and attribute tables
toctree = indent("\n".join([m.rst_name for m in sorted_members]))
starter = """
.. toctree::
:hidden:
{toctree}
.. class:: {name}
{rst_doc}""".format(toctree=toctree, name=get_type_name(cls), rst_doc=indent(cls.__doc__))
lines = [starter]
lines.append(indent(get_attribute_summary_table(sorted_members), 4))
lines.append(indent(get_method_summary_table(sorted_members), 4))
installation = get_installation(cls)
init_commands = [c for c in installation.commands if c.is_constructor]
if init_commands:
lines.append("\n.. _%s:\n" % get_cls_init_rst_label(cls)) # add rst label for internal reference
lines.append(get_command_def_rst(init_commands[0]))
return "\n".join(lines)
def write_py_rst_cls_file(root_path, cls):
installation = get_installation(cls)
entity_collection_name = installation.install_path.entity_collection_name
header = get_rst_cls_file_header(entity_collection_name, class_name=get_type_name(cls))
content = get_class_rst(cls)
folder = get_rst_folder_path(root_path, cls)
file_path = os.path.join(folder, "index.rst")
write_text_to_file(file_path, [header, content])
def write_py_rst_cls_file(root_path, cls):
installation = get_installation(cls)
entity_collection_name = installation.install_path.entity_collection_name
header = get_rst_cls_file_header(entity_collection_name, class_name=get_type_name(cls))
content = get_class_rst(cls)
folder = get_rst_folder_path(root_path, cls)
file_path = os.path.join(folder, "index.rst")
write_text_to_file(file_path, [header, content])
def get_entity_collection_name_and_install_path(obj, attr):
installation = get_installation(obj, None)
if installation:
entity_collection_name = installation.install_path.entity_collection_name
install_path = installation.install_path.full.replace(':', '-')
elif attr and has_entity_collection(attr):
entity_collection_name = get_entity_collection(attr)
install_path = ''
else:
raise RuntimeError("Unable to determine the entity collection for method %s" % attr)
return entity_collection_name, install_path
def get_entity_collection_name_and_install_path(obj, attr):
installation = get_installation(obj, None)
if installation:
entity_collection_name = installation.install_path.entity_collection_name
install_path = installation.install_path.full.replace(':', '-')
elif attr and has_entity_collection(attr):
entity_collection_name = get_entity_collection(attr)
install_path = ''
else:
raise RuntimeError(
"Unable to determine the entity collection for method %s" % attr)
return entity_collection_name, install_path
def get_entity_collection_name_and_install_path(obj, attr):
installation = get_installation(obj, None)
if installation:
entity_collection_name = installation.install_path.entity_collection_name
install_path = installation.install_path.full.replace(':', '-')
elif attr and has_entity_collection(attr):
entity_collection_name = get_entity_collection(attr)
install_path = ''
else:
# todo: fix me
entity_collection_name = "frames" # hack to put global items in frames, until we get a global area in the docs
install_path = ''
# raise RuntimeError("Unable to determine the entity collection for method %s" % attr)
return entity_collection_name, install_path
def check_loadable_class(cls, command_def):
error = None
installation = metaprog.get_installation(cls)
if not installation:
error = "class %s is not prepared for command installation" % cls
elif command_def.is_constructor and not installation.install_path.is_entity:
raise RuntimeError("API load error: special method '%s' may only be defined on entity classes, not on entity "
"member classes (i.e. only one slash allowed)." % command_def.full_name)
elif command_def.entity_type != installation.install_path.entity_type:
error = "%s is not the class's accepted command entity_type: %s" \
% (command_def.entity_type, installation.install_path.entity_type)
if error:
raise ValueError("API load error: Class %s cannot load command_def '%s'.\n%s"
% (cls.__name__, command_def.full_name, error))
def check_loadable_class(cls, command_def):
error = None
installation = metaprog.get_installation(cls)
if not installation:
error = "class %s is not prepared for command installation" % cls
elif command_def.is_constructor and not installation.install_path.is_entity:
raise RuntimeError("API load error: special method '%s' may only be defined on entity classes, not on entity "
"member classes (i.e. only one slash allowed)." % command_def.full_name)
elif command_def.entity_type != installation.install_path.entity_type:
error = "%s is not the class's accepted command entity_type: %s" \
% (command_def.entity_type, installation.install_path.entity_type)
if error:
raise ValueError("API load error: Class %s cannot load command_def '%s'.\n%s"
% (cls.__name__, command_def.full_name, error))
def get_index_ref(cls):
installation = get_installation(cls, None)
nested_level = len(
installation.install_path._intermediate_names) if installation else 0
return ("../" * nested_level) + 'index'
def get_index_ref(cls):
installation = get_installation(cls, None)
nested_level = len(installation.install_path._intermediate_names) if installation else 0
return ("../" * nested_level) + 'index'
