def get_third_party_dependencies(
config: Config, exclude: List[str] = util.SimpleFrozenList()) -> List[str]:
"""If the config includes references to registered functions that are
provided by third-party packages (spacy-transformers, other libraries), we
want to include them in meta["requirements"] so that the package specifies
them as dependencies and the user won't have to do it manually.
We do this by:
- traversing the config to check for registered function (@ keys)
- looking up the functions and getting their module
- looking up the module version and generating an appropriate version range
config (Config): The pipeline config.
exclude (list): List of packages to exclude (e.g. that already exist in meta).
RETURNS (list): The versioned requirements.
"""
own_packages = ("spacy", "spacy-legacy", "spacy-nightly", "thinc", "srsly")
distributions = util.packages_distributions()
funcs = defaultdict(set)
# We only want to look at runtime-relevant sections, not [training] or [initialize]
for section in ("nlp", "components"):
for path, value in util.walk_dict(config[section]):
if path[-1].startswith(
"@"): # collect all function references by registry
funcs[path[-1][1:]].add(value)
for component in config.get("components", {}).values():
if "factory" in component:
funcs["factories"].add(component["factory"])
modules = set()
lang = config["nlp"]["lang"]
for reg_name, func_names in funcs.items():
for func_name in func_names:
# Try the lang-specific version and fall back
try:
func_info = util.registry.find(reg_name,
lang + "." + func_name)
except RegistryError:
try:
func_info = util.registry.find(reg_name, func_name)
except RegistryError as regerr:
# lang-specific version being absent is not actually an issue
raise regerr from None
module_name = func_info.get("module") # type: ignore[attr-defined]
if module_name: # the code is part of a module, not a --code file
modules.add(
func_info["module"].split(".")[0]) # type: ignore[index]
dependencies = []
for module_name in modules:
if module_name in distributions:
dist = distributions.get(module_name)
if dist:
pkg = dist[0]
if pkg in own_packages or pkg in exclude:
continue
version = util.get_package_version(pkg)
version_range = util.get_minor_version_range(
version) # type: ignore[arg-type]
dependencies.append(f"{pkg}{version_range}")
return dependencies
def get_third_party_dependencies(
config: Config, exclude: List[str] = util.SimpleFrozenList()) -> List[str]:
"""If the config includes references to registered functions that are
provided by third-party packages (spacy-transformers, other libraries), we
want to include them in meta["requirements"] so that the package specifies
them as dependencies and the user won't have to do it manually.
We do this by:
- traversing the config to check for registered function (@ keys)
- looking up the functions and getting their module
- looking up the module version and generating an appropriate version range
config (Config): The pipeline config.
exclude (list): List of packages to exclude (e.g. that already exist in meta).
RETURNS (list): The versioned requirements.
"""
own_packages = ("spacy", "spacy-nightly", "thinc", "srsly")
distributions = util.packages_distributions()
funcs = defaultdict(set)
for path, value in util.walk_dict(config):
if path[-1].startswith(
"@"): # collect all function references by registry
funcs[path[-1][1:]].add(value)
for component in config.get("components", {}).values():
if "factory" in component:
funcs["factories"].add(component["factory"])
modules = set()
for reg_name, func_names in funcs.items():
sub_registry = getattr(util.registry, reg_name)
for func_name in func_names:
func_info = sub_registry.find(func_name)
module_name = func_info.get("module")
if module_name: # the code is part of a module, not a --code file
modules.add(func_info["module"].split(".")[0])
dependencies = []
for module_name in modules:
if module_name in distributions:
dist = distributions.get(module_name)
if dist:
pkg = dist[0]
if pkg in own_packages or pkg in exclude:
continue
version = util.get_package_version(pkg)
version_range = util.get_minor_version_range(version)
dependencies.append(f"{pkg}{version_range}")
return dependencies