def publish_extensions(extensions,
storage_subscription,
storage_account,
storage_container,
dist_dir='dist',
update_index=False,
yes=False):
heading('Publish Extensions')
require_azure_cli()
# rebuild the extensions
subheading('Building WHLs')
try:
shutil.rmtree(dist_dir)
except Exception as ex: # pylint: disable=broad-except
logger.debug("Unable to clear folder '%s'. Error: %s", dist_dir, ex)
build_extensions(extensions, dist_dir=dist_dir)
whl_files = find_files(dist_dir, '*.whl')
uploaded_urls = []
subheading('Uploading WHLs')
for whl_path in whl_files:
whl_file = os.path.split(whl_path)[-1]
# check if extension already exists unless user opted not to
if not yes:
command = 'az storage blob exists --subscription {} --account-name {} -c {} -n {}'.format(
storage_subscription, storage_account, storage_container,
whl_file)
exists = json.loads(cmd(command).result)['exists']
if exists:
if not prompt_y_n(
"{} already exists. You may need to bump the extension version. Replace?"
.format(whl_file),
default='n'):
logger.warning("Skipping '%s'...", whl_file)
continue
# upload the WHL file
command = 'az storage blob upload --subscription {} --account-name {} -c {} -n {} -f {}'.format(
storage_subscription, storage_account, storage_container, whl_file,
os.path.abspath(whl_path))
cmd(command, "Uploading '{}'...".format(whl_file))
command = 'az storage blob url --subscription {} --account-name {} -c {} -n {} -otsv'.format(
storage_subscription, storage_account, storage_container, whl_file)
url = cmd(command).result
logger.info(url)
uploaded_urls.append(url)
if update_index:
subheading('Updating Index')
update_extension_index(uploaded_urls)
subheading('Published')
display(uploaded_urls)
if not update_index:
logger.warning(
'You still need to update the index for your changes with `az extension update-index`.'
)
def _check_history_headings(mod_path):
history_path = os.path.join(mod_path, HISTORY_NAME)
source_path = None
destination_path = None
errors = []
with open(history_path, 'r') as f:
input_string = f.read()
_, pub = core.publish_programmatically(
source_class=io.StringInput,
source=input_string,
source_path=source_path,
destination_class=io.NullOutput,
destination=None,
destination_path=destination_path,
reader=None,
reader_name='standalone',
parser=None,
parser_name='restructuredtext',
writer=None,
writer_name='null',
settings=None,
settings_spec=None,
settings_overrides={},
config_section=None,
enable_exit_status=None)
# Check first heading is Release History
if pub.writer.document.children[0].rawsource != RELEASE_HISTORY_TITLE:
errors.append(
"Expected '{}' as first heading in HISTORY.rst".format(
RELEASE_HISTORY_TITLE))
all_versions = [
t['names'][0] for t in pub.writer.document.children if t['names']
]
# Check that no headings contain 'unreleased'. We don't require it any more
if any('unreleased' in v.lower() for v in all_versions):
errors.append(
"We no longer require 'unreleased' in headings. Use the appropriate version number instead."
)
# Check that the current package version has a history entry
if not all_versions:
errors.append(
"Unable to get versions from {}. Check formatting. e.g. there should be a new "
"line after the 'Release History' heading.".format(
history_path))
first_version_history = all_versions[0]
actual_version = cmd('python setup.py --version', cwd=mod_path)
# command can output warnings as well, so we just want the last line, which should have the version
actual_version = actual_version.result.splitlines()[-1].strip()
if first_version_history != actual_version:
errors.append(
"The topmost version in {} does not match version {} defined in setup.py."
.format(history_path, actual_version))
return errors
def __enter__(self):
if self.target_profile is None or self.target_profile == self.origin_profile:
display(
'The tests are set to run against current profile "{}"'.format(
self.origin_profile))
else:
result = cmd(
'az cloud update --profile {}'.format(self.target_profile),
'Switching to target profile "{}"...'.format(
self.target_profile))
if result.exit_code != 0:
raise CLIError(result.error.output.decode('utf-8'))
def _check_readme_render(mod_path):
errors = []
result = cmd('python setup.py check -r -s', cwd=mod_path)
if result.exit_code:
# this outputs some warnings we don't care about
error_lines = []
target_line = 'The following syntax errors were detected'
suppress = True
logger.debug(result.error.output)
# TODO: Checks for syntax errors but potentially not other things
for line in result.error.output.splitlines():
line = str(line).strip()
if not suppress and line:
error_lines.append(line)
if target_line in line:
suppress = False
errors.append(os.linesep.join(error_lines))
errors += _check_history_headings(mod_path)
return errors
def check_load_time(runs=3):
require_azure_cli()
heading('Module Load Performance')
regex = r"[^']*'([^']*)'[\D]*([\d\.]*)"
results = {TOTAL: []}
# Time the module loading X times
for i in range(0, runs + 1):
lines = cmd('az -h --debug', show_stderr=True).result
if i == 0:
# Ignore the first run since it can be longer due to *.pyc file compilation
continue
try:
lines = lines.decode().splitlines()
except AttributeError:
lines = lines.splitlines()
total_time = 0
for line in lines:
if line.startswith('DEBUG: Loaded module'):
matches = re.match(regex, line)
mod = matches.group(1)
val = float(matches.group(2)) * 1000
total_time = total_time + val
if mod in results:
results[mod].append(val)
else:
results[mod] = [val]
results[TOTAL].append(total_time)
passed_mods = {}
failed_mods = {}
mods = sorted(results.keys())
bubble_found = False
for mod in mods:
val = results[mod]
mean_val = mean(val)
stdev_val = pstdev(val)
threshold = THRESHOLDS.get(mod) or DEFAULT_THRESHOLD
statistics = {
'average': mean_val,
'stdev': stdev_val,
'threshold': threshold,
'values': val
}
if mean_val > threshold:
if not bubble_found and mean_val < 30:
# This temporary measure allows one floating performance
# failure up to 30 ms. See issue #6224 and #6218.
bubble_found = True
passed_mods[mod] = statistics
else:
failed_mods[mod] = statistics
else:
passed_mods[mod] = statistics
subheading('Results')
if failed_mods:
display('== PASSED MODULES ==')
display_table(passed_mods)
display('\nFAILED MODULES')
display_table(failed_mods)
raise CLIError("""
FAILED: Some modules failed. If values are close to the threshold, rerun. If values
are large, check that you do not have top-level imports like azure.mgmt or msrestazure
in any modified files.
""")
display('== PASSED MODULES ==')
display_table(passed_mods)
display('\nPASSED: Average load time all modules: {} ms'.format(
int(passed_mods[TOTAL]['average'])))
def test_setup(self):
from knack.prompting import NoTTYException
with self.assertRaises(NoTTYException):
cmd('setup')
def check_load_time(runs=3):
require_azure_cli()
heading('Module Load Performance')
regex = r"[^']*'(?P<mod>[^']*)'[\D]*(?P<val>[\d\.]*)"
results = {TOTAL: []}
# Time the module loading X times
for i in range(0, runs + 1):
lines = cmd('az -h --debug', show_stderr=True).result
if i == 0:
# Ignore the first run since it can be longer due to *.pyc file compilation
continue
try:
lines = lines.decode().splitlines()
except AttributeError:
lines = lines.splitlines()
total_time = 0
for line in lines:
if line.startswith('DEBUG: Loaded module'):
matches = re.match(regex, line)
mod = matches.group('mod')
val = float(matches.group('val')) * 1000
total_time = total_time + val
if mod in results:
results[mod].append(val)
else:
results[mod] = [val]
results[TOTAL].append(total_time)
passed_mods = {}
failed_mods = {}
def _claim_higher_threshold(val):
avail_thresholds = {k: v for k, v in THRESHOLDS.items() if v}
new_threshold = None
for threshold in sorted(avail_thresholds):
if val < threshold:
THRESHOLDS[threshold] = THRESHOLDS[threshold] - 1
new_threshold = threshold
break
return new_threshold
mods = sorted(results.keys())
for mod in mods:
val = results[mod]
mean_val = mean(val)
stdev_val = pstdev(val)
threshold = TOTAL_THRESHOLD if mod == TOTAL else DEFAULT_THRESHOLD
statistics = {
'average': mean_val,
'stdev': stdev_val,
'threshold': threshold,
'values': val
}
if mean_val > threshold:
# claim a threshold exception if available
new_threshold = _claim_higher_threshold(mean_val)
if new_threshold:
statistics['threshold'] = new_threshold
passed_mods[mod] = statistics
else:
failed_mods[mod] = statistics
else:
passed_mods[mod] = statistics
subheading('Results')
if failed_mods:
display('== PASSED MODULES ==')
display_table(passed_mods)
display('\nFAILED MODULES')
display_table(failed_mods)
raise CLIError("""
FAILED: Some modules failed. If values are close to the threshold, rerun. If values
are large, check that you do not have top-level imports like azure.mgmt or msrestazure
in any modified files.
""")
display('== PASSED MODULES ==')
display_table(passed_mods)
display('\nPASSED: Average load time all modules: {} ms'.format(
int(passed_mods[TOTAL]['average'])))
def test_setup(self):
from azdev.utilities import test_cmd as cmd
cmd('setup')
def test_style(self):
cmd('style')
def current_profile():
return cmd('az cloud show --query profile -otsv', show_stderr=False).result
def test_test(self):
cmd('test')
