def verify_default_modules(args):
errors_list = []
cli_deps = get_cli_dependencies(args.build_folder)
all_command_modules = get_command_modules_paths(include_prefix=True)
if not cli_deps:
print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_SETUP_PY), file=sys.stderr)
sys.exit(1)
for modname, _ in all_command_modules:
if modname not in cli_deps:
errors_list.append("{} is not included to be installed by default! Modify {}.".format(modname, AZURE_CLI_SETUP_PY))
if errors_list:
print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_SETUP_PY))
print('\n'.join(errors_list), file=sys.stderr)
sys.exit(1)
else:
print('Verified default modules list successfully.', file=sys.stderr)
def verify_default_modules(args):
errors_list = []
cli_deps = get_cli_dependencies(args.build_folder)
all_command_modules = get_command_modules_paths(include_prefix=True)
if not cli_deps:
print('Unable to get the CLI dependencies for {}'.format(AZURE_CLI_SETUP_PY), file=sys.stderr)
sys.exit(1)
for modname, _ in all_command_modules:
if modname in cli_deps and modname in MODULES_TO_EXCLUDE:
errors_list.append("{} is a dependency of azure-cli BUT is marked as should be excluded.".format(modname))
if modname not in cli_deps and modname not in MODULES_TO_EXCLUDE:
errors_list.append("{} is not included to be installed by default! If this is a mistake, modify {}. "
"Otherwise, modify this script ({}) to exclude the module.".format(modname,
AZURE_CLI_SETUP_PY,
__file__))
if errors_list:
print_heading('Errors whilst verifying default modules list in {}!'.format(AZURE_CLI_SETUP_PY))
print('\n'.join(errors_list), file=sys.stderr)
sys.exit(1)
else:
print('Verified default modules list successfully.', file=sys.stderr)
def verify_default_modules(args):
errors_list = []
cli_deps = get_cli_dependencies(args.build_folder)
all_command_modules = get_command_modules_paths(include_prefix=True)
if not cli_deps:
print('Unable to get the CLI dependencies for {}'.format(
AZURE_CLI_SETUP_PY),
file=sys.stderr)
sys.exit(1)
for modname, _ in all_command_modules:
if modname not in cli_deps:
errors_list.append(
"{} is not included to be installed by default! Modify {}.".
format(modname, AZURE_CLI_SETUP_PY))
if errors_list:
print_heading(
'Errors whilst verifying default modules list in {}!'.format(
AZURE_CLI_SETUP_PY))
print('\n'.join(errors_list), file=sys.stderr)
sys.exit(1)
else:
print('Verified default modules list successfully.', file=sys.stderr)
def install_package(path_to_package, package_name, dist_dir):
print_heading('Installing {}'.format(path_to_package))
cmd = 'python -m pip install --upgrade {} --find-links file://{}'.format(package_name, dist_dir)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(path_to_package), f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(path_to_package))
def install_package(path_to_package, package_name, dist_dir):
print_heading('Installing {}'.format(path_to_package))
cmd = 'python -m pip install --upgrade {} --find-links file://{}'.format(package_name, dist_dir)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(path_to_package), f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(path_to_package))
def install_pip_package(package_name):
print_heading('Installing {}'.format(package_name))
cmd = 'python -m pip install {}'.format(package_name)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(package_name),
f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(package_name))
def build_package(path_to_package, dist_dir):
print_heading('Building {}'.format(path_to_package))
path_to_setup = os.path.join(path_to_package, 'setup.py')
set_version(path_to_setup)
cmd_success = exec_command('python setup.py bdist_wheel -d {0}'.format(dist_dir), cwd=path_to_package)
if not cmd_success:
print_heading('Error building {}!'.format(path_to_package), f=sys.stderr)
sys.exit(1)
print_heading('Built {}'.format(path_to_package))
def build_package(path_to_package, dist_dir):
print_heading('Building {}'.format(path_to_package))
path_to_setup = os.path.join(path_to_package, 'setup.py')
set_version(path_to_setup)
cmd_success = exec_command('python setup.py bdist_wheel -d {0}'.format(dist_dir), cwd=path_to_package)
if not cmd_success:
print_heading('Error building {}!'.format(path_to_package), f=sys.stderr)
sys.exit(1)
print_heading('Built {}'.format(path_to_package))
def verify_packages():
# tmp dir to store all the built packages
built_packages_dir = tempfile.mkdtemp()
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(
include_prefix=True)
# STEP 1:: Build the packages
for name, path in all_modules:
build_package(path, built_packages_dir)
# STEP 2:: Install the CLI and dependencies
azure_cli_modules_path = next(path for name, path in all_modules
if name == 'azure-cli')
install_package(azure_cli_modules_path, 'azure-cli', built_packages_dir)
# Install the remaining command modules
for name, fullpath in all_command_modules:
install_package(fullpath, name, built_packages_dir)
# STEP 3:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'],
stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [
dist.key for dist in pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)
]
print('Installed command modules', installed_command_modules)
missing_modules = \
set([name for name, fullpath in all_command_modules]) - set(installed_command_modules)
if missing_modules:
print_heading(
'Error: The following modules were not installed successfully',
f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
print_heading('OK')
def verify_packages():
# tmp dir to store all the built packages
built_packages_dir = tempfile.mkdtemp()
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(include_prefix=True)
# STEP 1:: Build the packages
for name, path in all_modules:
build_package(path, built_packages_dir)
# STEP 2:: Install the CLI and dependencies
azure_cli_modules_path = next(path for name, path in all_modules if name == 'azure-cli')
install_package(azure_cli_modules_path, 'azure-cli', built_packages_dir)
# Install the remaining command modules
for name, fullpath in all_command_modules:
install_package(fullpath, name, built_packages_dir)
# STEP 3:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'], stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [dist.key for dist in
pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)]
print('Installed command modules', installed_command_modules)
missing_modules = \
set([name for name, fullpath in all_command_modules]) - set(installed_command_modules)
if missing_modules:
print_heading('Error: The following modules were not installed successfully', f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
print_heading('OK')
def install_pip_package(package_name, preview=False, extra_link=None):
print_heading('Installing {}'.format(package_name))
preview_str = "--pre" if preview else ""
if extra_link:
extra_link_str = "--extra-index-url extra_link --no-cache-dir"
else:
extra_link_str = ""
cmd = 'python -m pip install {} {} {}'.format(preview_str, package_name,
extra_link_str)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(package_name),
f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(package_name))
def install_package(path_to_package, package_name, dist_dir):
#sys.path.remove(path_to_package)
print("deleting {}".format(
os.path.join(path_to_package, 'azure_cli_batch_extensions.egg-info')))
shutil.rmtree(
os.path.join(path_to_package, 'azure_cli_batch_extensions.egg-info'))
print("deleting {}".format(os.path.join(path_to_package, 'build')))
shutil.rmtree(os.path.join(path_to_package, 'build'))
print_heading('Installing {}'.format(path_to_package))
cmd = 'python -m pip install --upgrade {} --find-links file://{}'.format(
package_name, dist_dir)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(path_to_package),
f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(path_to_package))
def install_package(path_to_package, package_name, dist_dir):
egg_path = os.path.join(
path_to_package, '{}.egg-info'.format(package_name.replace('-', '_')))
print("deleting {}".format(egg_path))
try:
shutil.rmtree(egg_path)
except Exception as exp:
print("Couldn't delete: {}".format(exp))
print("deleting {}".format(os.path.join(path_to_package, 'build')))
try:
shutil.rmtree(os.path.join(path_to_package, 'build'))
except Exception as exp:
print("Couldn't delete: {}".format(exp))
print_heading('Installing {}'.format(path_to_package))
cmd = 'python -m pip install --upgrade {} --find-links file://{}'.format(
package_name, dist_dir)
cmd_success = exec_command(cmd)
if not cmd_success:
print_heading('Error installing {}!'.format(path_to_package),
f=sys.stderr)
sys.exit(1)
print_heading('Installed {}'.format(path_to_package))
def verify_packages():
# tmp dir to store all the built packages
built_packages_dir = tempfile.mkdtemp()
all_modules = automation_path.get_all_module_paths()
# STEP 1:: Install the CLI and dependencies by pip
install_pip_package('azure-cli')
# STEP 2:: Build the packages
for name, path in all_modules:
build_package(path, built_packages_dir)
# STEP 3:: Install Batch Extensions and validate
install_package(all_modules[0][1], all_modules[0][0], built_packages_dir)
try:
importlib.import_module('azext.batch')
except ImportError as err:
print("Unable to import {}".format(name))
print(err)
sys.exit(1)
# STEP 4:: Add CLI extension wheel to CLI
# try:
# extension_whl = os.path.join(built_packages_dir, 'azure_batch_cli_extensions-1000.0.0-py2.py3-none-any.whl')
# az_output = subprocess.check_output(['az', 'extension', 'add', '--source', extension_whl, '--debug'], stderr=subprocess.STDOUT,
# universal_newlines=True)
# success = 'Successfully installed azure-batch-cli-extensions-1000.0.0' in az_output
# print(az_output, file=sys.stderr)
# except subprocess.CalledProcessError as err:
# success = False
# print(err, file=sys.stderr)
# STEP 5:: Verify extension loading correctly
try:
az_output = subprocess.check_output(['az', '--debug'],
stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
if success:
success = 'Loaded extension \'azure-batch-cli-extensions\'' in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_modules = [
dist.key for dist in pip.get_installed_distributions(local_only=True)
]
print('Installed command modules', installed_modules)
missing_modules = \
set([all_modules[0][0]]) - set(installed_modules)
if missing_modules:
print_heading(
'Error: The following modules were not installed successfully',
f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
print_heading('OK')
def verify_packages(built_packages_dir):
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(include_prefix=True)
modules_missing_manifest_in = [name for name, path in all_modules if not os.path.isfile(os.path.join(path, 'MANIFEST.in'))]
if modules_missing_manifest_in:
print_heading('Error: The following modules are missing the MANIFEST.in file.')
print(modules_missing_manifest_in)
sys.exit(1)
# STEP 1:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'], stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
# STEP 2:: Run -h on each command
print('Running --help on all commands.')
from azure.cli.core.application import Configuration
config = Configuration()
all_commands = list(config.get_command_table())
command_results = []
p = multiprocessing.Pool(multiprocessing.cpu_count())
for i, res in enumerate(p.imap_unordered(run_help_on_command_without_err, all_commands, 10), 1):
sys.stderr.write('{}/{} \t'.format(i, len(all_commands)))
sys.stderr.flush()
command_results.append(res)
p.close()
p.join()
if not all(command_results):
print_heading('Error running --help on commands in the CLI!', f=sys.stderr)
sys.exit(1)
print('OK.')
# STEP 3:: Determine if any modules failed to install
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [dist.key for dist in
pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)]
print('Installed command modules', installed_command_modules)
missing_modules = set([name for name, fullpath in all_command_modules]) - set(installed_command_modules) - \
EXCLUDE_MODULES
if missing_modules:
print_heading('Error: The following modules were not installed successfully', f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
# STEP 4:: Verify the wheels that get produced
print_heading('Verifying wheels...')
invalid_wheels = []
for wheel_path in glob.glob(os.path.join(built_packages_dir, '*.whl')):
# Verify all non-nspkg wheels
if 'nspkg' not in wheel_path and not _valid_wheel(wheel_path):
invalid_wheels.append(wheel_path)
if invalid_wheels:
print_heading('Error: The following wheels are invalid', f=sys.stderr)
print(invalid_wheels, file=sys.stderr)
print(VALID_WHEEL_HELP, file=sys.stderr)
sys.exit(1)
print_heading('Verified wheels successfully.')
print_heading('OK')
def verify_packages(built_packages_dir):
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(
include_prefix=True)
modules_missing_manifest_in = [
name for name, path in all_modules
if not os.path.isfile(os.path.join(path, 'MANIFEST.in'))
]
if modules_missing_manifest_in:
print_heading(
'Error: The following modules are missing the MANIFEST.in file.')
print(modules_missing_manifest_in)
sys.exit(1)
# STEP 1:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'],
stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
# STEP 2:: Run -h on each command
print('Running --help on all commands.')
from azure.cli.core.application import Configuration
config = Configuration()
all_commands = list(config.get_command_table())
command_results = []
p = multiprocessing.Pool(multiprocessing.cpu_count())
for i, res in enumerate(
p.imap_unordered(run_help_on_command_without_err, all_commands,
10), 1):
sys.stderr.write('{}/{} \t'.format(i, len(all_commands)))
sys.stderr.flush()
command_results.append(res)
p.close()
p.join()
if not all(command_results):
print_heading('Error running --help on commands in the CLI!',
f=sys.stderr)
sys.exit(1)
print('OK.')
# STEP 3:: Determine if any modules failed to install
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [
dist.key for dist in pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)
]
print('Installed command modules', installed_command_modules)
missing_modules = set([name for name, fullpath in all_command_modules]) - set(installed_command_modules) - \
EXCLUDE_MODULES
if missing_modules:
print_heading(
'Error: The following modules were not installed successfully',
f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
# STEP 4:: Verify the wheels that get produced
print_heading('Verifying wheels...')
invalid_wheels = []
for wheel_path in glob.glob(os.path.join(built_packages_dir, '*.whl')):
# Verify all non-nspkg wheels
if 'nspkg' not in wheel_path and not _valid_wheel(wheel_path):
invalid_wheels.append(wheel_path)
if invalid_wheels:
print_heading('Error: The following wheels are invalid', f=sys.stderr)
print(invalid_wheels, file=sys.stderr)
print(VALID_WHEEL_HELP, file=sys.stderr)
sys.exit(1)
print_heading('Verified wheels successfully.')
print_heading('OK')
def verify_packages():
# tmp dir to store all the built packages
built_packages_dir = tempfile.mkdtemp()
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(include_prefix=True)
modules_missing_manifest_in = [name for name, path in all_modules if not os.path.isfile(os.path.join(path, 'MANIFEST.in'))]
if modules_missing_manifest_in:
print_heading('Error: The following modules are missing the MANIFEST.in file.')
print(modules_missing_manifest_in)
sys.exit(1)
# STEP 1:: Build the packages
for name, path in all_modules:
build_package(path, built_packages_dir)
# STEP 2:: Install the CLI and dependencies
azure_cli_modules_path = next(path for name, path in all_modules if name == 'azure-cli')
install_package(azure_cli_modules_path, 'azure-cli', built_packages_dir)
# Install the remaining command modules
for name, fullpath in all_command_modules:
install_package(fullpath, name, built_packages_dir)
# STEP 3:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'], stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
# STEP 4:: Run -h on each command
print('Running --help on all commands.')
from azure.cli.core.application import Configuration
config = Configuration()
all_commands = list(config.get_command_table())
pool_size = 5
chunk_size = 10
command_results = []
p = multiprocessing.Pool(pool_size)
prev_percent = 0
for i, res in enumerate(p.imap_unordered(run_help_on_command_without_err, all_commands, chunk_size), 1):
command_results.append(res)
cur_percent = int((i/len(all_commands))*100)
if cur_percent != prev_percent:
print('{}% complete'.format(cur_percent), file=sys.stderr)
prev_percent = cur_percent
p.close()
p.join()
if not all(command_results):
print_heading('Error running --help on commands in the CLI!', f=sys.stderr)
sys.exit(1)
print('OK.')
# STEP 5:: Determine if any modules failed to install
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [dist.key for dist in
pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)]
print('Installed command modules', installed_command_modules)
missing_modules = \
set([name for name, fullpath in all_command_modules]) - set(installed_command_modules)
if missing_modules:
print_heading('Error: The following modules were not installed successfully', f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
# STEP 6:: Verify the wheels that get produced
print_heading('Verifying wheels...')
invalid_wheels = []
for wheel_path in glob.glob(os.path.join(built_packages_dir, '*.whl')):
# Verify all non-nspkg wheels
if 'nspkg' not in wheel_path and not _valid_wheel(wheel_path):
invalid_wheels.append(wheel_path)
if invalid_wheels:
print_heading('Error: The following wheels are invalid', f=sys.stderr)
print(invalid_wheels, file=sys.stderr)
print(VALID_WHEEL_HELP, file=sys.stderr)
sys.exit(1)
print_heading('Verified wheels successfully.')
print_heading('OK')
def verify_packages():
# tmp dir to store all the built packages
built_packages_dir = tempfile.mkdtemp()
all_modules = automation_path.get_all_module_paths()
all_command_modules = automation_path.get_command_modules_paths(
include_prefix=True)
modules_missing_manifest_in = [
name for name, path in all_modules
if not os.path.isfile(os.path.join(path, 'MANIFEST.in'))
]
if modules_missing_manifest_in:
print_heading(
'Error: The following modules are missing the MANIFEST.in file.')
print(modules_missing_manifest_in)
sys.exit(1)
# STEP 1:: Build the packages
for name, path in all_modules:
build_package(path, built_packages_dir)
# STEP 2:: Install the CLI and dependencies
azure_cli_modules_path = next(path for name, path in all_modules
if name == 'azure-cli')
install_package(azure_cli_modules_path, 'azure-cli', built_packages_dir)
# Install the remaining command modules
for name, fullpath in all_command_modules:
install_package(fullpath, name, built_packages_dir)
# STEP 3:: Validate the installation
try:
az_output = subprocess.check_output(['az', '--debug'],
stderr=subprocess.STDOUT,
universal_newlines=True)
success = 'Error loading command module' not in az_output
print(az_output, file=sys.stderr)
except subprocess.CalledProcessError as err:
success = False
print(err, file=sys.stderr)
if not success:
print_heading('Error running the CLI!', f=sys.stderr)
sys.exit(1)
# STEP 4:: Run -h on each command
print('Running --help on all commands.')
from azure.cli.core.application import Configuration
config = Configuration()
all_commands = list(config.get_command_table())
pool_size = 5
chunk_size = 10
command_results = []
p = multiprocessing.Pool(pool_size)
prev_percent = 0
for i, res in enumerate(
p.imap_unordered(run_help_on_command_without_err, all_commands,
chunk_size), 1):
command_results.append(res)
cur_percent = int((i / len(all_commands)) * 100)
if cur_percent != prev_percent:
print('{}% complete'.format(cur_percent), file=sys.stderr)
prev_percent = cur_percent
p.close()
p.join()
if not all(command_results):
print_heading('Error running --help on commands in the CLI!',
f=sys.stderr)
sys.exit(1)
print('OK.')
# STEP 5:: Determine if any modules failed to install
pip.utils.pkg_resources = imp.reload(pip.utils.pkg_resources)
installed_command_modules = [
dist.key for dist in pip.get_installed_distributions(local_only=True)
if dist.key.startswith(COMMAND_MODULE_PREFIX)
]
print('Installed command modules', installed_command_modules)
missing_modules = \
set([name for name, fullpath in all_command_modules]) - set(installed_command_modules)
if missing_modules:
print_heading(
'Error: The following modules were not installed successfully',
f=sys.stderr)
print(missing_modules, file=sys.stderr)
sys.exit(1)
# STEP 6:: Verify the wheels that get produced
print_heading('Verifying wheels...')
invalid_wheels = []
for wheel_path in glob.glob(os.path.join(built_packages_dir, '*.whl')):
# Verify all non-nspkg wheels
if 'nspkg' not in wheel_path and not _valid_wheel(wheel_path):
invalid_wheels.append(wheel_path)
if invalid_wheels:
print_heading('Error: The following wheels are invalid', f=sys.stderr)
print(invalid_wheels, file=sys.stderr)
print(VALID_WHEEL_HELP, file=sys.stderr)
sys.exit(1)
print_heading('Verified wheels successfully.')
print_heading('OK')
