def _load_external_cards():
# Load external card packages
card_packages = _get_external_card_packages()
if not card_packages:
return []
external_cards = {}
card_arr = []
# Load cards from all external packages.
for package in card_packages:
try:
cards = package.CARDS
# Ensure that types match.
if not type(cards) == list:
continue
except AttributeError as e:
_ext_debug("Card import failed with error : %s" % str(e))
continue
else:
for c in cards:
if not isinstance(c, type) or not issubclass(c, MetaflowCard):
# every card should only be inheriting a MetaflowCard
continue
if not getattr(c, "type", None):
# todo Warn user of non existant `type` in MetaflowCard
continue
if c.type in external_cards:
# todo Warn user of duplicate card
continue
# external_cards[c.type] = c
_ext_debug("Adding card of type: %s" % str(c.type))
card_arr.append(c)
return card_arr
def _get_external_card_packages(with_paths=False):
"""
Safely extract all exteranl card modules
Args:
with_paths (bool, optional): setting `with_paths=True` will result in a list of tuples: `[( mf_extensions_parent_path , relative_path_to_module, module)]`. setting false will return a list of modules Defaults to False.
Returns:
`list` of `ModuleType` or `list` of `tuples` where each tuple if of the form (mf_extensions_parent_path , relative_path_to_module, ModuleType)
"""
import importlib
available_card_modules = []
for m in get_modules("plugins.cards"):
# Iterate submodules of metaflow_extensions.X.plugins.cards
# For example metaflow_extensions.X.plugins.cards.monitoring
card_packages = []
for fndx, card_mod, ispkg_c in iter_namespace(m.module):
try:
if not ispkg_c:
continue
cm = importlib.import_module(card_mod)
_ext_debug("Importing card package %s" % card_mod)
if with_paths:
card_packages.append((fndx.path, cm))
else:
card_packages.append(cm)
except Exception as e:
_ext_debug(
"External Card Module Import Exception \n\n %s \n\n %s" %
(str(e), traceback.format_exc()))
if with_paths:
card_packages = [
(
os.path.abspath(os.path.join(
pth,
"../../../../")), # parent path to metaflow_extensions
os.path.join(
EXT_PKG,
os.path.relpath(m.__path__[0],
os.path.join(pth, "../../../")),
), # construct relative path to parent.
m,
) for pth, m in card_packages
]
available_card_modules.extend(card_packages)
return available_card_modules
def _get_external_card_packages():
"""
Safely extract all external card modules
Returns:
`list` of `ModuleType`
"""
import importlib
# Caching card related modules.
global _CARD_MODULES
if len(_CARD_MODULES) > 0:
return _CARD_MODULES
for m in get_modules("plugins.cards"):
card_packages = []
# condition checks if it is not a namespace package or is a regular package.
if getattr(m.module, "__file__", None):
# This supports the following cases
# - a namespace package support with mfextinit_X.py
# - a regular package support
card_packages.append(m.module)
else:
# This is to support current system where you have a namespace package and then sub packages that have __init__.py
for _, card_mod, ispkg_c in iter_namespace(m.module):
# Iterate submodules of metaflow_extensions.X.plugins.cards
# For example metaflow_extensions.X.plugins.cards.monitoring
try:
if not ispkg_c:
continue
cm = importlib.import_module(card_mod)
_ext_debug("Importing card package %s" % card_mod)
card_packages.append(cm)
except Exception as e:
_ext_debug(
"External Card Module Import Exception \n\n %s \n\n %s"
% (str(e), traceback.format_exc())
)
_CARD_MODULES.extend(card_packages)
return _CARD_MODULES
# as it is likely to not work.
_override_modules = []
_tl_modules = []
try:
_modules_to_import = get_modules("toplevel")
for m in _modules_to_import:
override_module = m.module.__dict__.get("module_overrides", None)
if override_module is not None:
_override_modules.append(".".join(
[EXT_PKG, m.tl_package, "toplevel", override_module]))
tl_module = m.module.__dict__.get("toplevel", None)
if tl_module is not None:
_tl_modules.append(".".join(
[EXT_PKG, m.tl_package, "toplevel", tl_module]))
_ext_debug("Got overrides to load: %s" % _override_modules)
_ext_debug("Got top-level imports: %s" % _tl_modules)
except Exception as e:
_ext_debug("Error in importing toplevel/overrides: %s" % e)
# Load overrides now that we have them (in the proper order)
for m in _override_modules:
extension_module = load_module(m)
if extension_module:
# We load only modules
tl_package = m.split(".")[1]
lazy_load_aliases(alias_submodules(extension_module, tl_package, None))
# Utilities
from .multicore_utils import parallel_imap_unordered, parallel_map
from .metaflow_profile import profile
try:
from metaflow.extension_support import get_modules, multiload_all, _ext_debug
_modules_to_import = get_modules("plugins")
multiload_all(_modules_to_import, "plugins", globals())
# Build an ordered list
_ext_plugins = {k: v[0] for k, v in _expected_extensions.items()}
for m in _modules_to_import:
for k, v in _expected_extensions.items():
module_override = m.module.__dict__.get(k)
if module_override is not None:
v[1](_ext_plugins[k], module_override)
except Exception as e:
_ext_debug(
"\tWARNING: ignoring all plugins due to error during import: %s" % e)
_ext_plugins = {k: v[0] for k, v in _expected_extensions.items()}
_ext_debug("\tWill import the following plugins: %s" % str(_ext_plugins))
def get_plugin_cli():
# it is important that CLIs are not imported when
# __init__ is imported. CLIs may use e.g.
# parameters.add_custom_parameters which requires
# that the flow is imported first
# Add new CLI commands in this list
from . import package_cli
from .aws.batch import batch_cli
from .kubernetes import kubernetes_cli