def build_cache(self, modules, excluded_packages, non_binary_packages):
wheels_folder = os.path.join(self.cache_folder, 'wheels')
wheels_install_folder = os.path.join(self.cache_folder, 'dependencies')
cache_zip_file = os.path.join(self.cache_folder, 'cache.zip')
cache_metadata_file = os.path.join(self.cache_folder, 'metadata.json')
packages = DependencyManager.get_dependency_packages_list(
modules, excluded_packages)
packages_root = os.path.join('.python_packages', 'lib', 'python3.6',
'site-packages')
if not DependencyManager.check_cache(cache_metadata_file,
cache_zip_file, packages):
cache_pkg = AzurePythonPackageArchive()
self.log.info(
"Cached packages not found or requirements were changed.")
# If cache check fails, wipe all previous wheels, installations etc
if os.path.exists(self.cache_folder):
self.log.info("Removing cache folder...")
shutil.rmtree(self.cache_folder)
self.log.info("Preparing non binary wheels...")
DependencyManager.prepare_non_binary_wheels(
non_binary_packages, wheels_folder)
self.log.info("Downloading wheels...")
DependencyManager.download_wheels(packages, wheels_folder)
self.log.info("Installing wheels...")
DependencyManager.install_wheels(wheels_folder,
wheels_install_folder)
for root, _, files in os.walk(wheels_install_folder):
arc_prefix = os.path.relpath(root, wheels_install_folder)
for f in files:
dest_path = os.path.join(packages_root, arc_prefix, f)
if f.endswith('.pyc') or f.endswith('.c'):
continue
f_path = os.path.join(root, f)
cache_pkg.add_file(f_path, dest_path)
self.log.info('Saving cache zip file...')
cache_pkg.close()
with open(cache_zip_file, 'wb') as fout:
fout.write(cache_pkg.get_stream().read())
self.log.info("Removing temporary folders...")
shutil.rmtree(wheels_folder)
shutil.rmtree(wheels_install_folder)
self.log.info("Updating metadata file...")
DependencyManager.create_cache_metadata(cache_metadata_file,
cache_zip_file, packages)
return cache_zip_file
def test_create_cache_metadata(self):
bench = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(bench))
tmp_metadata = os.path.join(bench, 'metadata.json')
DependencyManager.create_cache_metadata(tmp_metadata, self.test_zip,
self.test_packages)
with open(self.test_metadata, 'rt') as f:
test_json = json.load(f)
with open(tmp_metadata, 'rt') as f:
tmp_json = json.load(f)
self.assertTrue(test_json == tmp_json)
def build(self, policy, modules, non_binary_packages, excluded_packages, queue_name=None,):
wheels_folder = os.path.join(self.cache_folder, 'wheels')
wheels_install_folder = os.path.join(self.cache_folder, 'dependencies')
packages = \
DependencyManager.get_dependency_packages_list(modules, excluded_packages)
if not DependencyManager.check_cache(self.cache_folder, wheels_install_folder, packages):
self.log.info("Cached packages not found or requirements were changed.")
# If cache check fails, wipe all previous wheels, installations etc
if os.path.exists(self.cache_folder):
self.log.info("Removing cache folder...")
shutil.rmtree(self.cache_folder)
self.log.info("Preparing non binary wheels...")
DependencyManager.prepare_non_binary_wheels(non_binary_packages, wheels_folder)
self.log.info("Downloading wheels...")
DependencyManager.download_wheels(packages, wheels_folder)
self.log.info("Installing wheels...")
DependencyManager.install_wheels(wheels_folder, wheels_install_folder)
self.log.info("Updating metadata file...")
DependencyManager.create_cache_metadata(self.cache_folder,
wheels_install_folder,
packages)
for root, _, files in os.walk(wheels_install_folder):
arc_prefix = os.path.relpath(root, wheels_install_folder)
for f in files:
dest_path = os.path.join(arc_prefix, f)
if f.endswith('.pyc') or f.endswith('.c'):
continue
f_path = os.path.join(root, f)
self.pkg.add_file(f_path, dest_path)
exclude = os.path.normpath('/cache/') + os.path.sep
self.pkg.add_modules(lambda f: (exclude in f),
*[m.replace('-', '_') for m in modules])
# add config and policy
self._add_functions_required_files(policy, queue_name)
# generate and add auth
s = local_session(Session)
self.pkg.add_contents(dest=self.name + '/auth.json', contents=s.get_functions_auth_string())
def test_get_installed_distributions(self):
d = DependencyManager.get_dependency_packages_list(
['c7n-azure', 'c7n-azure'], ['azure-cli-core'])
self.assertTrue('azure-cli-core' not in d)
self.assertTrue('adal' in d)
self.assertEqual(sorted(d), d)
# Remove versions from all packages & make sure there is no duplicates in the list
regex = "^[^<>~=]*"
d_no_versions = [re.match(regex, p).group(0) for p in d]
self.assertEqual(len(d), len(set(d_no_versions)))
def test_check_hash(self):
self.assertFalse(
DependencyManager.check_cache(self.test_metadata_wrong,
self.test_zip, self.test_packages))
self.assertFalse(
DependencyManager.check_cache(self.test_metadata,
self.test_zip_wrong,
self.test_packages))
self.assertFalse(
DependencyManager.check_cache(self.test_metadata, self.test_zip,
['wrong', 'wrong2']))
self.assertFalse(
DependencyManager.check_cache(self.test_metadata,
self.test_metadata,
self.test_packages))
self.assertTrue(
DependencyManager.check_cache(self.test_metadata, self.test_zip,
self.test_packages))
def test_get_installed_distributions(self):
d = DependencyManager.get_dependency_packages_list(
['c7n-azure', 'c7n-azure'], ['azure-cli-core'])
# expected dependencies
self.assertTrue('adal' in d)
# excluded packages are missing
self.assertTrue('azure-cli-core' not in d)
# dependencies that are substrings of another are includes
self.assertTrue('applicationinsights' in d)
self.assertTrue('azure-mgmt-applicationinsights' in d)
# dependencies are sorted
self.assertEqual(sorted(d), d)
# Remove versions from all packages & make sure there is no duplicates in the list
regex = "^[^<>~=]*"
d_no_versions = [re.match(regex, p).group(0) for p in d]
self.assertEqual(len(d), len(set(d_no_versions)))
def test_get_string_hash(self):
self.assertEqual(
DependencyManager._get_string_hash(' '.join(self.test_packages)),
'1189b389ffc75d3a3174b6c63dee03fc')
def test_get_file_hash(self):
self.assertEqual(DependencyManager._get_file_hash(self.test_zip),
'EqAMFyrJXL+/+kEgji2hHQESjSHDTm4/SQZjwVdVcgg=')
def build(
self,
policy,
modules,
non_binary_packages,
excluded_packages,
queue_name=None,
):
wheels_folder = os.path.join(self.cache_folder, 'wheels')
wheels_install_folder = os.path.join(self.cache_folder, 'dependencies')
cache_zip_file = os.path.join(self.cache_folder, 'cache.zip')
cache_metadata_file = os.path.join(self.cache_folder, 'metadata.json')
packages = \
DependencyManager.get_dependency_packages_list(modules, excluded_packages)
if not DependencyManager.check_cache(cache_metadata_file,
cache_zip_file, packages):
cache_pkg = PythonPackageArchive()
self.log.info(
"Cached packages not found or requirements were changed.")
# If cache check fails, wipe all previous wheels, installations etc
if os.path.exists(self.cache_folder):
self.log.info("Removing cache folder...")
shutil.rmtree(self.cache_folder)
self.log.info("Preparing non binary wheels...")
DependencyManager.prepare_non_binary_wheels(
non_binary_packages, wheels_folder)
self.log.info("Downloading wheels...")
DependencyManager.download_wheels(packages, wheels_folder)
self.log.info("Installing wheels...")
DependencyManager.install_wheels(wheels_folder,
wheels_install_folder)
for root, _, files in os.walk(wheels_install_folder):
arc_prefix = os.path.relpath(root, wheels_install_folder)
for f in files:
dest_path = os.path.join(arc_prefix, f)
if f.endswith('.pyc') or f.endswith('.c'):
continue
f_path = os.path.join(root, f)
cache_pkg.add_file(f_path, dest_path)
self.log.info('Saving cache zip file...')
cache_pkg.close()
with open(cache_zip_file, 'wb') as fout:
fout.write(cache_pkg.get_stream().read())
self.log.info("Removing temporary folders...")
shutil.rmtree(wheels_folder)
shutil.rmtree(wheels_install_folder)
self.log.info("Updating metadata file...")
DependencyManager.create_cache_metadata(cache_metadata_file,
cache_zip_file, packages)
self.pkg = PythonPackageArchive(cache_file=cache_zip_file)
exclude = os.path.normpath('/cache/') + os.path.sep
self.pkg.add_modules(lambda f: (exclude in f),
[m.replace('-', '_') for m in modules])
# add config and policy
self._add_functions_required_files(policy, queue_name)
def build(
self,
policy,
modules,
non_binary_packages,
excluded_packages,
queue_name=None,
):
wheels_folder = os.path.join(self.cache_folder, 'wheels')
wheels_install_folder = os.path.join(self.cache_folder, 'dependencies')
packages = \
DependencyManager.get_dependency_packages_list(modules, excluded_packages)
if not DependencyManager.check_cache(self.cache_folder,
wheels_install_folder, packages):
self.log.info(
"Cached packages not found or requirements were changed.")
# If cache check fails, wipe all previous wheels, installations etc
if os.path.exists(self.cache_folder):
self.log.info("Removing cache folder...")
shutil.rmtree(self.cache_folder)
self.log.info("Preparing non binary wheels...")
DependencyManager.prepare_non_binary_wheels(
non_binary_packages, wheels_folder)
self.log.info("Downloading wheels...")
DependencyManager.download_wheels(packages, wheels_folder)
self.log.info("Installing wheels...")
DependencyManager.install_wheels(wheels_folder,
wheels_install_folder)
self.log.info("Updating metadata file...")
DependencyManager.create_cache_metadata(self.cache_folder,
wheels_install_folder,
packages)
for root, _, files in os.walk(wheels_install_folder):
arc_prefix = os.path.relpath(root, wheels_install_folder)
for f in files:
dest_path = os.path.join(arc_prefix, f)
if f.endswith('.pyc') or f.endswith('.c'):
continue
f_path = os.path.join(root, f)
self.pkg.add_file(f_path, dest_path)
exclude = os.path.normpath('/cache/') + os.path.sep
self.pkg.add_modules(lambda f: (exclude in f),
*[m.replace('-', '_') for m in modules])
# add config and policy
self._add_functions_required_files(policy, queue_name)
# generate and add auth
s = local_session(Session)
self.pkg.add_contents(dest=self.name + '/auth.json',
contents=s.get_functions_auth_string())