def test_clear_path_importer_cache_and_modules_retain_namespace(self):
# First import common_module from _customer_deps_path as customer_mod
sys.path.insert(0, self._customer_deps_path)
import common_module as customer_mod # NoQA
self.assertIn('common_module', sys.modules)
self.assertEqual(
customer_mod.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)
# Clean up cache
DependencyManager._clear_path_importer_cache_and_modules(
self._customer_deps_path
)
self.assertNotIn('common_module', sys.modules)
# Try import common_module from _worker_deps_path as worker_mod
sys.path.insert(0, self._worker_deps_path)
# Ensure new import is from _worker_deps_path
import common_module as worker_mod # NoQA
self.assertIn('common_module', sys.modules)
self.assertEqual(
worker_mod.package_location,
os.path.join(self._worker_deps_path, 'common_module')
)
def test_reload_all_namespaces_from_customer_deps(self):
"""The test simulates a linux consumption environment where the worker
transits from placeholder mode to specialized mode. In a very typical
scenario, the nested azure.functions library (with common azure)
namespace needs to be switched from worker_deps to customer_Deps.
"""
self._initialize_scenario()
# Ensure the nested_module is imported from _worker_deps_path
DependencyManager.use_worker_dependencies()
import common_namespace.nested_module # NoQA
self.assertEqual(common_namespace.nested_module.__version__, 'worker')
# At placeholder specialization from function_environment_reload
DependencyManager.prioritize_customer_dependencies(
self._customer_func_path
)
# Now the nested_module should be imported from customer dependency
import common_namespace.nested_module # NoQA
self.assertIn(self._customer_deps_path, sys.path_importer_cache)
self.assertEqual(
common_namespace.__path__[0],
os.path.join(self._customer_deps_path, 'common_namespace')
)
self.assertEqual(common_namespace.nested_module.__version__, 'customer')
# Check if the order matches expectation
self._assert_path_order(sys.path, [
self._customer_deps_path,
self._worker_deps_path,
self._customer_func_path,
])
def test_add_to_sys_path_no_duplication(self):
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
path_count = len(list(filter(
lambda x: x == self._customer_deps_path, sys.path
)))
self.assertEqual(path_count, 1)
def test_add_to_sys_path_import_module(self):
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
import common_module # NoQA
self.assertEqual(
common_module.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)
def test_remove_module_cache_with_namespace_remain(self):
# Create common_module namespace
sys.path.append(self._customer_deps_path)
import common_module # NoQA
# Ensure namespace remains after module cache is removed
DependencyManager._remove_module_cache(self._customer_deps_path)
self.assertIsNotNone(common_module)
def test_add_to_sys_path_allow_resolution_from_import_statement(self):
"""The standard Python import mechanism allows deriving a specific
module in an import statement, e.g.
from azure import functions # OK
"""
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
from common_namespace import nested_module # NoQA
self.assertEqual(nested_module.__version__, 'customer')
def test_remove_module_cache(self):
# First import the common_module and create a sys.modules cache
sys.path.append(self._customer_deps_path)
import common_module # NoQA
self.assertIn('common_module', sys.modules)
# Ensure the module cache will be remove
DependencyManager._remove_module_cache(self._customer_deps_path)
self.assertNotIn('common_module', sys.modules)
def test_remove_from_sys_path_should_remove_related_module(self):
# Import a common_module from customer deps will create a module import
# cache in sys.module
sys.path.insert(0, self._customer_deps_path)
import common_module # NoQA
self.assertIn('common_module', sys.modules)
# Remove sys.path_importer_cache
DependencyManager._remove_from_sys_path(self._customer_deps_path)
self.assertNotIn('common_module', sys.modules)
def test_add_to_sys_path_disallow_module_resolution_from_namespace(self):
"""The standard Python import mechanism does not allow deriving a
specific module from a namespace without the import statement, e.g.
import azure
azure.functions # Error: module 'azure' has not attribute 'functions'
"""
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
import common_namespace # NoQA
with self.assertRaises(AttributeError):
common_namespace.nested_module
def test_add_to_sys_path_import_namespace_path(self):
"""Check if a common_namespace can be loaded after adding its path
into sys.path
"""
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
import common_namespace # NoQA
self.assertEqual(len(common_namespace.__path__), 1)
self.assertEqual(
common_namespace.__path__[0],
os.path.join(self._customer_deps_path, 'common_namespace')
)
def test_use_worker_dependencies_default_python_36_37_38_39(self):
# Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# The common_module cannot be imported since feature is disabled
DependencyManager.use_worker_dependencies()
with self.assertRaises(ImportError):
import common_module # NoQA
def test_prioritize_customer_dependencies_default_python_36_37_38_39(self):
# Feature should be disabled in Python 3.6, 3.7, 3.8 and 3.9
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the common_module is imported from _customer_deps_path
DependencyManager.prioritize_customer_dependencies()
with self.assertRaises(ImportError):
import common_module # NoQA
def test_remove_from_sys_path_should_remove_nested_module(self):
"""When a nested module is imported into a namespace, the sys.modules
should cache it. After calling the remove_from_sys_path, the nested
module should be removed from sys.modules
"""
sys.path.insert(0, self._customer_deps_path)
import common_namespace.nested_module # NoQA
self.assertIn('common_namespace.nested_module', sys.modules)
# Remove from sys.modules via _remove_from_sys_path
DependencyManager._remove_from_sys_path(self._customer_deps_path)
self.assertNotIn('common_namespace.nested_module', sys.modules)
def test_prioritize_customer_dependencies_disable(self):
# Setup app settings
os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false'
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the common_module is imported from _customer_deps_path
DependencyManager.prioritize_customer_dependencies()
with self.assertRaises(ImportError):
import common_module # NoQA
def test_clear_path_importer_cache_and_modules(self):
# Ensure sys.path_importer_cache and sys.modules cache is cleared
sys.path.insert(0, self._customer_deps_path)
import common_module # NoQA
self.assertIn('common_module', sys.modules)
# Clear out cache
DependencyManager._clear_path_importer_cache_and_modules(
self._customer_deps_path
)
# Ensure cache is cleared
self.assertNotIn('common_module', sys.modules)
def test_use_worker_dependencies_disable(self):
# Setup app settings
os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'false'
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# The common_module cannot be imported since feature is disabled
DependencyManager.use_worker_dependencies()
with self.assertRaises(ImportError):
import common_module # NoQA
def test_prioritize_customer_dependencies_default_python_310(self):
# Feature should be enabled in Python 3.10 by default
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the common_module is imported from _customer_deps_path
DependencyManager.prioritize_customer_dependencies()
import common_module # NoQA
self.assertEqual(
common_module.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)
def test_prioritize_customer_dependencies_from_working_directory(self):
self._initialize_scenario()
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the func_specific_module is imported from _customer_func_path
DependencyManager.prioritize_customer_dependencies()
import func_specific_module # NoQA
self.assertEqual(
func_specific_module.package_location,
os.path.join(self._customer_func_path, 'func_specific_module')
)
def test_get_worker_deps_path_with_no_worker_sys_path(self):
result = DependencyManager._get_worker_deps_path()
azf_spec = importlib.util.find_spec('azure.functions')
worker_parent = os.path.abspath(
os.path.join(os.path.dirname(azf_spec.origin), '..', '..')
)
self.assertEqual(result.lower(), worker_parent.lower())
def test_use_worker_dependencies(self):
# Setup app settings
os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true'
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the common_module is imported from _worker_deps_path
DependencyManager.use_worker_dependencies()
import common_module # NoQA
self.assertEqual(
common_module.package_location,
os.path.join(self._worker_deps_path, 'common_module')
)
def test_get_cx_deps_path_in_script_root_with_sys_path_windows(self):
# Test for Windows Core Tools Environment
sys.path.append('C:\\FunctionApp\\sites\\lib\\site-packages')
os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp'
result = DependencyManager._get_cx_deps_path()
self.assertEqual(result,
'C:\\FunctionApp\\sites\\lib\\site-packages')
def test_get_cx_deps_path_in_script_root_with_sys_path_linux(self):
# Test for Python 3.7+ Azure Environment
sys.path.append('/home/site/wwwroot/.python_packages/sites/lib/'
'site-packages/')
os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot'
result = DependencyManager._get_cx_deps_path()
self.assertEqual(result, '/home/site/wwwroot/.python_packages/sites/'
'lib/site-packages/')
def test_get_worker_deps_path_without_worker_path(self, mock):
# Test when worker path is not provided
mock.find_spec.return_value = None
sys.path.append('/home/site/wwwroot')
result = DependencyManager._get_worker_deps_path()
worker_parent = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..')
)
self.assertEqual(result.lower(), worker_parent.lower())
def test_get_worker_deps_path_from_windows_core_tools(self):
# Test for Windows Core Tools Environment
sys.path.append('C:\\Users\\user\\AppData\\Roaming\\npm\\'
'node_modules\\azure-functions-core-tools\\bin\\'
'workers\\python\\3.6\\WINDOWS\\X64')
result = DependencyManager._get_worker_deps_path()
self.assertEqual(result,
'C:\\Users\\user\\AppData\\Roaming\\npm\\'
'node_modules\\azure-functions-core-tools\\bin\\'
'workers\\python\\3.6\\WINDOWS\\X64')
def test_initialize_in_linux_dedicated(self):
os.environ['AzureWebJobsScriptRoot'] = '/home/site/wwwroot'
sys.path.extend([
'/home/site/wwwroot',
'/home/site/wwwroot/.python_packages/lib/site-packages',
'/azure-functions-host/workers/python/3.7/LINUX/X64'
])
DependencyManager.initialize()
self.assertEqual(
DependencyManager.cx_deps_path,
'/home/site/wwwroot/.python_packages/lib/site-packages'
)
self.assertEqual(
DependencyManager.cx_working_dir,
'/home/site/wwwroot',
)
self.assertEqual(
DependencyManager.worker_deps_path,
'/azure-functions-host/workers/python/3.7/LINUX/X64'
)
def main(req: func.HttpRequest) -> func.HttpResponse:
"""This function is an HttpTrigger to check if the modules are loaded from
customer's dependencies. We have mock a .python_packages/ folder in
this e2e test function app which contains the following stub package:
protobuf==3.9.0
grpc==1.35.0
If the version we check is the same as the one in local .python_packages/,
that means the isolate worker dependencies are working as expected.
"""
result = {
"sys.path": list(sys.path),
"dependency_manager": {
"cx_deps_path": dm._get_cx_deps_path(),
"cx_working_dir": dm._get_cx_working_dir(),
"worker_deps_path": dm._get_worker_deps_path(),
},
"libraries": {
"func.version": func.__version__,
"func.file": func.__file__,
"proto.expected.version": "3.9.0",
"proto.version": proto.__version__,
"proto.file": proto.__file__,
"grpc.expected.version": "1.35.0",
"grpc.version": grpc.__version__,
"grpc.file": grpc.__file__,
},
"environments": {
"PYTHON_ISOLATE_WORKER_DEPENDENCIES":
(os.getenv('PYTHON_ISOLATE_WORKER_DEPENDENCIES')),
"AzureWebJobsScriptRoot":
os.getenv('AzureWebJobsScriptRoot'),
"PYTHONPATH":
os.getenv('PYTHONPATH'),
"HOST_VERSION":
os.getenv('HOST_VERSION')
}
}
return func.HttpResponse(json.dumps(result))
def test_reload_all_modules_from_customer_deps(self):
"""The test simulates a linux consumption environment where the worker
transits from placeholder mode to specialized worker with customer's
dependencies. First the worker will use worker's dependencies for its
own modules. After worker init request, it starts adding customer's
library path into sys.path (e.g. .python_packages/). The final step
is in environment reload where the worker is fully specialized,
reloading all libraries from customer's package.
"""
self._initialize_scenario()
# Ensure the common_module is imported from _worker_deps_path
DependencyManager.use_worker_dependencies()
import common_module # NoQA
self.assertEqual(
common_module.package_location,
os.path.join(self._worker_deps_path, 'common_module')
)
# At placeholder specialization from function_environment_reload
DependencyManager.prioritize_customer_dependencies(
self._customer_func_path
)
# Now the module should be imported from customer dependency
import common_module # NoQA
self.assertIn(self._customer_deps_path, sys.path_importer_cache)
self.assertEqual(
common_module.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)
# Check if the order matches expectation
self._assert_path_order(sys.path, [
self._customer_deps_path,
self._worker_deps_path,
self._customer_func_path,
])
def test_prioritize_customer_dependencies(self):
# Setup app settings
os.environ['PYTHON_ISOLATE_WORKER_DEPENDENCIES'] = 'true'
# Setup paths
DependencyManager.worker_deps_path = self._worker_deps_path
DependencyManager.cx_deps_path = self._customer_deps_path
DependencyManager.cx_working_dir = self._customer_func_path
# Ensure the common_module is imported from _customer_deps_path
DependencyManager.prioritize_customer_dependencies()
import common_module # NoQA
self.assertEqual(
common_module.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)
# Check if the sys.path order matches the expected order
self._assert_path_order(sys.path, [
self._customer_deps_path,
self._worker_deps_path,
self._customer_func_path,
])
def test_initialize_in_windows_core_tools(self):
os.environ['AzureWebJobsScriptRoot'] = 'C:\\FunctionApp'
sys.path.extend([
'C:\\Users\\user\\AppData\\Roaming\\npm\\'
'node_modules\\azure-functions-core-tools\\bin\\'
'workers\\python\\3.6\\WINDOWS\\X64',
'C:\\FunctionApp\\.venv38\\lib\\site-packages',
'C:\\FunctionApp'
])
DependencyManager.initialize()
self.assertEqual(
DependencyManager.cx_deps_path,
'C:\\FunctionApp\\.venv38\\lib\\site-packages'
)
self.assertEqual(
DependencyManager.cx_working_dir,
'C:\\FunctionApp',
)
self.assertEqual(
DependencyManager.worker_deps_path,
'C:\\Users\\user\\AppData\\Roaming\\npm\\node_modules\\'
'azure-functions-core-tools\\bin\\workers\\python\\3.6\\WINDOWS'
'\\X64'
)
def test_add_to_sys_path_importer_cache_reloaded(self):
# First import the common module from worker_deps_path
DependencyManager._add_to_sys_path(self._worker_deps_path, True)
import common_module # NoQA
self.assertIn(self._worker_deps_path, sys.path_importer_cache)
self.assertEqual(
common_module.package_location,
os.path.join(self._worker_deps_path, 'common_module')
)
# Mock that the customer's script are running in a different module
# (e.g. HttpTrigger/__init__.py)
del sys.modules['common_module']
del common_module
# Import the common module from customer_deps_path
# Customer should only see their own module
DependencyManager._add_to_sys_path(self._customer_deps_path, True)
import common_module # NoQA
self.assertIn(self._customer_deps_path, sys.path_importer_cache)
self.assertEqual(
common_module.package_location,
os.path.join(self._customer_deps_path, 'common_module')
)