def _import_by_name(name: str) -> Tuple[Any, Any, str]:
"""Import a Python object given its full name."""
try:
name_parts = name.split('.')
# # 20200429 first try whether it is a full module name
# try:
# mod = import_module(name)
# parent_name = '.'.join(name_parts[:-1])
# if parent_name:
# parent = import_module(parent_name)
# else:
# parent = None
# return parent, mod, name
# except ImportError:
# pass
# try first interpret `name` as MODNAME.OBJ
modname = '.'.join(name_parts[:-1])
if modname:
try:
mod = import_module(modname)
return getattr(mod, name_parts[-1]), mod, modname
except (ImportError, IndexError, AttributeError):
pass
# ... then as MODNAME, MODNAME.OBJ1, MODNAME.OBJ1.OBJ2, ...
last_j = 0
modname = None
for j in reversed(range(1, len(name_parts) + 1)):
last_j = j
modname = '.'.join(name_parts[:j])
try:
import_module(modname)
except ImportError as e:
logger.info("Failed to import %s : %s", modname, e)
continue
if modname in sys.modules:
break
if last_j < len(name_parts):
parent = None
obj = sys.modules[modname]
for obj_name in name_parts[last_j:]:
parent = obj
obj = getattr(obj, obj_name)
return obj, parent, modname
else:
return sys.modules[modname], None, modname
except (ValueError, ImportError, AttributeError, KeyError) as e:
raise ImportError(*e.args)
def get_template_field(env, fullname):
"""
Gets template fields for specific operator class.
:param fullname: Full path to operator class.
For example: ``airflow.providers.google.cloud.operators.vision.CloudVisionCreateProductSetOperator``
:return: List of template field
:rtype: list[str]
"""
modname, classname = fullname.rsplit(".", 1)
try:
with mock(env.config.autodoc_mock_imports):
mod = import_module(modname)
except ImportError:
raise RoleException(f"Error loading {modname} module.")
clazz = getattr(mod, classname)
if not clazz:
raise RoleException(
f"Error finding {classname} class in {modname} module.")
template_fields = getattr(clazz, "template_fields")
if not template_fields:
raise RoleException(
f"Could not find the template fields for {classname} class in {modname} module."
)
return list(template_fields)
def get_template_field(env, fullname):
"""
Gets template fields for specific operator class.
:param fullname: Full path to operator class.
For example: ``airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator``
:return: List of template field
:rtype: list[str]
"""
modname, classname = fullname.rsplit(".", 1)
try:
with mock(env.config.autodoc_mock_imports):
mod = import_module(modname)
except ImportError:
raise RoleException("Error loading %s module." % (modname, ))
clazz = getattr(mod, classname)
if not clazz:
raise RoleException("Error finding %s class in %s module." % (classname, modname))
template_fields = getattr(clazz, "template_fields")
if not template_fields:
raise RoleException(
"Could not find the template fields for %s class in %s module." % (classname, modname)
)
return list(template_fields)
def get_template_field(env, fullname):
"""
Gets template fields for specific operator class.
:param fullname: Full path to operator class.
For example: ``airflow.contrib.operators.gcp_vision_operator.CloudVisionProductSetCreateOperator``
:return: List of template field
:rtype: list[str]
"""
modname, classname = fullname.rsplit(".", 1)
try:
with mock(env.config.autodoc_mock_imports):
mod = import_module(modname)
except ImportError:
raise RoleException("Error loading %s module." % (modname, ))
clazz = getattr(mod, classname)
if not clazz:
raise RoleException("Error finding %s class in %s module." %
(classname, modname))
template_fields = getattr(clazz, "template_fields")
if not template_fields:
raise RoleException(
"Could not find the template fields for %s class in %s module." %
(classname, modname))
return list(template_fields)
def _import_by_name(name: str,
grouped_exception: bool = True) -> Tuple[Any, Any, str]:
"""Import a Python object given its full name."""
errors: List[BaseException] = []
try:
name_parts = name.split('.')
# try first interpret `name` as MODNAME.OBJ
modname = '.'.join(name_parts[:-1])
if modname:
try:
mod = import_module(modname)
return getattr(mod, name_parts[-1]), mod, modname
except (ImportError, IndexError, AttributeError) as exc:
errors.append(exc.__cause__ or exc)
# ... then as MODNAME, MODNAME.OBJ1, MODNAME.OBJ1.OBJ2, ...
last_j = 0
modname = None
for j in reversed(range(1, len(name_parts) + 1)):
last_j = j
modname = '.'.join(name_parts[:j])
try:
import_module(modname)
except ImportError as exc:
errors.append(exc.__cause__ or exc)
if modname in sys.modules:
break
if last_j < len(name_parts):
parent = None
obj = sys.modules[modname]
for obj_name in name_parts[last_j:]:
parent = obj
obj = getattr(obj, obj_name)
return obj, parent, modname
else:
return sys.modules[modname], None, modname
except (ValueError, ImportError, AttributeError, KeyError) as exc:
errors.append(exc)
if grouped_exception:
raise ImportExceptionGroup('', errors)
else:
raise ImportError(*exc.args) from exc
def _import_by_name(name):
# type: (str) -> Tuple[Any, Any, str]
"""Import a Python object given its full name."""
try:
name_parts = name.split('.')
# try first interpret `name` as MODNAME.OBJ
modname = '.'.join(name_parts[:-1])
if modname:
try:
mod = import_module(modname)
return getattr(mod, name_parts[-1]), mod, modname
except (ImportError, IndexError, AttributeError):
pass
# ... then as MODNAME, MODNAME.OBJ1, MODNAME.OBJ1.OBJ2, ...
last_j = 0
modname = None
for j in reversed(range(1, len(name_parts) + 1)):
last_j = j
modname = '.'.join(name_parts[:j])
try:
import_module(modname)
except ImportError:
continue
if modname in sys.modules:
break
if last_j < len(name_parts):
parent = None
obj = sys.modules[modname]
for obj_name in name_parts[last_j:]:
parent = obj
obj = getattr(obj, obj_name)
return obj, parent, modname
else:
return sys.modules[modname], None, modname
except (ValueError, ImportError, AttributeError, KeyError) as e:
raise ImportError(*e.args)
def _import_by_name(name):
# type: (str) -> Tuple[Any, Any, unicode]
"""Import a Python object given its full name."""
try:
name_parts = name.split('.')
# try first interpret `name` as MODNAME.OBJ
modname = '.'.join(name_parts[:-1])
if modname:
try:
mod = import_module(modname)
return getattr(mod, name_parts[-1]), mod, modname
except (ImportError, IndexError, AttributeError):
pass
# ... then as MODNAME, MODNAME.OBJ1, MODNAME.OBJ1.OBJ2, ...
last_j = 0
modname = None
for j in reversed(range(1, len(name_parts) + 1)):
last_j = j
modname = '.'.join(name_parts[:j])
try:
import_module(modname)
except ImportError:
continue
if modname in sys.modules:
break
if last_j < len(name_parts):
parent = None
obj = sys.modules[modname]
for obj_name in name_parts[last_j:]:
parent = obj
obj = getattr(obj, obj_name)
return obj, parent, modname
else:
return sys.modules[modname], None, modname
except (ValueError, ImportError, AttributeError, KeyError) as e:
raise ImportError(*e.args)
def get_modules(obj: Any) -> Tuple[List[str], List[str]]:
items: List[str] = []
for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__):
fullname = name + '.' + modname
try:
module = import_module(fullname)
if module and hasattr(module, '__sphinx_mock__'):
continue
except ImportError:
pass
items.append(fullname)
public = [x for x in items if not x.split('.')[-1].startswith('_')]
return public, items
def get_package_modules(pkgname):
"""Returns a list of module names within the given package."""
if pkgname in ignore_modules:
return []
spec = importlib.util.find_spec(pkgname)
if not spec:
logger.warning("Failed to find module {0}".format(pkgname))
return []
path = spec.submodule_search_locations
if not path:
# This is not a package, but a module.
# (Fun fact: if we don't return here, we will start importing all the
# modules on sys.path, which will have all sorts of hilarious effects
# like reading out the Zen of Python and opening xkcd #353 in the web
# browser.)
return []
names = []
for importer, modname, ispkg in pkgutil.iter_modules(path):
fullname = pkgname + '.' + modname
if fullname in ignore_modules:
continue
# Try importing the module; if we can't, then don't add it to the list.
try:
import_module(fullname)
except ImportError:
logger.exception("Failed to import {0}".format(fullname))
continue
names.append(fullname)
return names
def run(self) -> Sequence[nodes.Node]: # type: ignore
"""
Process the content of the directive.
"""
plugin: str = self.arguments[0]
if not self.content:
warnings.warn("No codes specified")
return []
module = import_module(plugin)
codes: List[Tuple[str, str]] = []
for code in self.content:
if code.strip():
try:
description = getattr(module, code)
if description.startswith(code):
description = description[len(code):]
codes.append((code, description.strip()))
except AttributeError:
warnings.warn(f"No such code {code!r}")
if not codes:
warnings.warn("No codes specified")
return []
targetid = f'flake8codes-{self.env.new_serialno("flake8codes"):d}'
targetnode = nodes.section(ids=[targetid])
table = tabulate.tabulate(codes,
headers=["Code", "Description"],
tablefmt="rst")
content = '\n' + table.replace('\t', " ") + '\n'
view = StringList(content.split('\n'))
table_node = nodes.paragraph(rawsource=content)
self.state.nested_parse(view, self.content_offset, table_node)
table_node_purger.add_node(self.env, table_node, targetnode,
self.lineno)
return [table_node]
def import_object_new(
modname: str,
objpath: List[str],
objtype: str = "",
attrgetter: Callable[[Any, str], Any] = safe_getattr,
warningiserror: bool = False,
) -> Any:
import imp
import types
print(
"----------------------------------------hack---------------------------------"
)
# print(f'modname: {modname} objpath: {objpath}')
# module = imp.new_module(modname)
global_ldr = globals()["global_ldr"]
class_path = modname.split(".")
base_obj_name = class_path.pop(0)
base_obj = global_ldr.get_functions()[base_obj_name]
# print(global_ldr.get_functions())
module = imp.new_module(base_obj_name)
# sys.modules[base_obj_name] = module
# for path in class_path:
# cur = getattr(base_obj,item)
parent = base_obj
# base_obj=getattr(base_obj,path)
for item in dir(base_obj):
cur = getattr(base_obj, item)
if isinstance(cur, contentPlugin):
setattr(module, item, cur)
print(f"register {base_obj_name}.{item}")
return [module, None, modname, module]
# parent=None
# for path in class_path:
# parent=base_obj
# base_obj=getattr(base_obj,path)
# pprint(dir(base_obj))
# func = getattr(module, func_name)
# if isinstance(func, types.FunctionType):
if objpath:
logger.debug("[autodoc] from %s import %s", modname, ".".join(objpath))
else:
logger.debug("[autodoc] import %s", modname)
try:
# module = None
exc_on_importing = None
objpath = list(objpath)
while module is None:
try:
module = import_module(modname, warningiserror=warningiserror)
logger.debug("[autodoc] import %s => %r", modname, module)
except ImportError as exc:
logger.debug("[autodoc] import %s => failed", modname)
exc_on_importing = exc
if "." in modname:
# retry with parent module
modname, name = modname.rsplit(".", 1)
objpath.insert(0, name)
else:
raise
obj = module
parent = None
object_name = None
for attrname in objpath:
parent = obj
logger.debug("[autodoc] getattr(_, %r)", attrname)
mangled_name = mangle(obj, attrname)
obj = attrgetter(obj, mangled_name)
logger.debug("[autodoc] => %r", obj)
object_name = attrname
print([module, parent, object_name, obj])
return [module, parent, object_name, obj]
except (AttributeError, ImportError) as exc:
if isinstance(exc, AttributeError) and exc_on_importing:
# restore ImportError
exc = exc_on_importing
if objpath:
errmsg = "autodoc: failed to import %s %r from module %r" % (
objtype,
".".join(objpath),
modname,
)
else:
errmsg = "autodoc: failed to import %s %r" % (objtype, modname)
if isinstance(exc, ImportError):
# import_module() raises ImportError having real exception obj and
# traceback
real_exc, traceback_msg = exc.args
if isinstance(real_exc, SystemExit):
errmsg += (
"; the module executes module level statement "
"and it might call sys.exit()."
)
elif isinstance(real_exc, ImportError) and real_exc.args:
errmsg += (
"; the following exception was raised:\n%s" % real_exc.args[0]
)
else:
errmsg += (
"; the following exception was raised:\n%s" % traceback_msg
)
else:
errmsg += (
"; the following exception was raised:\n%s" % traceback.format_exc()
)
logger.debug(errmsg)
raise ImportError(errmsg) from exc