def test_access_token_or_client_secret_must_be_set(logger: logger):
with pytest.raises(InterruptExecution) as execinfo:
init_client(dict(), CONFIG)
assert str(
execinfo.value
) == 'Authentication to Azure requires a client secret or an access token'
def test_credential_bad(spcred):
cred = MagicMock()
spcred.return_value = cred
init_client(SECRETS_BAD_CLOUD, CONFIG)
spcred.assert_called_with(client_id=SECRETS_BAD_CLOUD.get("client_id"),
secret=SECRETS_BAD_CLOUD.get('client_secret'),
tenant=SECRETS_BAD_CLOUD.get('tenant_id'),
cloud_environment=__get_cloud_env_by_name(
SECRETS_BAD_CLOUD.get('azure_cloud')))
def restart_vmss(filter: str = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Restart a virtual machine scale set instance at random.
Parameters
----------
filter : str
Filter the virtual machine scale set. If the filter is omitted all
virtual machine scale sets in the subscription will be selected as
potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
"""
logger.debug("Start restart_vmss: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = choose_vmss_at_random(filter, configuration, secrets)
vmss_instance = choose_vmss_instance_at_random(vmss, configuration,
secrets)
logger.debug("Restarting instance: {}".format(vmss_instance['name']))
client = init_client(secrets, configuration)
client.virtual_machine_scale_set_vms.restart(vmss['resourceGroup'],
vmss['name'],
vmss_instance['instanceId'])
def delete_vmss(filter: str = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Delete a virtual machine scale set instance at random.
**Be aware**: Deleting a VMSS instance is an invasive action. You will not
be able to recover the VMSS instance once you deleted it.
Parameters
----------
filter : str
Filter the virtual machine scale set. If the filter is omitted all
virtual machine scale sets in the subscription will be selected as
potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
"""
logger.debug("Start delete_vmss: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = choose_vmss_at_random(filter, configuration, secrets)
vmss_instance = choose_vmss_instance_at_random(vmss, configuration,
secrets)
logger.debug("Deleting instance: {}".format(vmss_instance['name']))
client = init_client(secrets, configuration)
client.virtual_machine_scale_set_vms.delete(vmss['resourceGroup'],
vmss['name'],
vmss_instance['instanceId'])
def delete_webapp(filter: str = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Delete a web app at random.
***Be aware**: Deleting a web app is an invasive action. You will not be
able to recover the web app once you deleted it.
Parameters
----------
filter : str
Filter the web apps. If the filter is omitted all web apps in
the subscription will be selected as potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
"""
logger.debug("Start delete_webapp: configuration='{}', filter='{}'".format(
configuration, filter))
choice = __fetch_webapp_at_random(filter, configuration, secrets)
logger.debug("Deleting web app: {}".format(choice['name']))
client = init_client(secrets, configuration)
client.web_apps.delete(choice['resourceGroup'], choice['name'])
def run(resource_group: str, compute: dict, timeout: int, parameters: dict,
secrets, configuration):
client = init_client(secrets, configuration)
compute_type = compute.get('type').lower()
if compute_type == RES_TYPE_VMSS_VM.lower():
poller = client.virtual_machine_scale_set_vms.run_command(
resource_group, compute['scale_set'], compute['instance_id'],
parameters)
elif compute_type == RES_TYPE_VM.lower():
poller = client.virtual_machines.run_command(resource_group,
compute['name'],
parameters)
else:
msg = "Trying to run a command for the unknown resource type '{}'" \
.format(compute.get('type'))
raise InterruptExecution(msg)
result = poller.result(timeout) # Blocking till executed
if result and result.value:
logger.debug(result.value[0].message) # stdout/stderr
else:
raise FailedActivity("Operation did not finish properly."
" You may consider increasing timeout setting.")
def stop_vmss(filter: str = None,
instance_criteria: Iterable[Mapping[str, any]] = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Stops instances from the filtered scale set either at random or by
a defined instance criteria.
Parameters
----------
filter : str
Filter the virtual machine scale set. If the filter is omitted all
virtual machine scale sets in the subscription will be selected as
potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
instance_criteria : Iterable[Mapping[str, any]]
Allows specification of criteria for selection of a given virtual
machine scale set instance. If the instance_criteria is omitted,
an instance will be chosen at random. All of the criteria within each
item of the Iterable must match, i.e. AND logic is applied.
The first item with all matching criterion will be used to select the
instance.
Criteria example:
[
{"name": "myVMSSInstance1"},
{
"name": "myVMSSInstance2",
"instanceId": "2"
}
{"instanceId": "3"},
]
If the instances include two items. One with name = myVMSSInstance4
and instanceId = 2. The other with name = myVMSSInstance2 and
instanceId = 3. The criteria {"instanceId": "3"} will be the first
match since both the name and the instanceId did not match on the
first criteria.
"""
logger.debug("Starting stop_vmss: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = fetch_vmss(filter, configuration, secrets)
vmss_records = Records()
for scale_set in vmss:
instances_records = Records()
instances = fetch_instances(scale_set, instance_criteria,
configuration, secrets)
for instance in instances:
logger.debug("Stopping instance: {}".format(instance['name']))
client = init_client(secrets, configuration)
client.virtual_machine_scale_set_vms.power_off(
scale_set['resourceGroup'], scale_set['name'],
instance['instance_id'])
instances_records.add(cleanse.vmss_instance(instance))
scale_set['virtualMachines'] = instances_records.output()
vmss_records.add(cleanse.vmss(scale_set))
return vmss_records.output_as_dict('resources')
def test_init_client_public_token_credentials(cmclient, aadcred):
client = MagicMock()
cmclient.return_value = client
cred = MagicMock()
aadcred.return_value = cred
init_client(SECRETS_PUBLIC_TOKEN, CONFIG)
aadcred.assert_called_with(
dict(accessToken=SECRETS_PUBLIC_TOKEN.get('access_token')),
client_id=SECRETS_PUBLIC_TOKEN.get('client_id'))
cmclient.assert_called_with(
credentials=ANY,
subscription_id=CONFIG['azure']['subscription_id'],
base_url=__get_cloud_env_by_name(
SECRETS_PUBLIC.get("azure_cloud")).endpoints.resource_manager)
def stop_vmss(filter: str = None,
configuration: Configuration = None,
instance_criteria: Iterable[Mapping[str, any]] = None,
secrets: Secrets = None):
"""
Stop a virtual machine scale set instance at random.
Parameters
----------
filter : str
Filter the virtual machine scale set. If the filter is omitted all
virtual machine scale sets in the subscription will be selected as
potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
instance_criteria : Iterable[Mapping[str, any]]
Allows specification of criteria for selection of a given virtual
machine scale set instance. If the instance_criteria is omitted,
an instance will be chosen at random. All of the criteria within each
item of the Iterable must match, i.e. AND logic is applied.
The first item with all matching criterion will be used to select the
instance.
Criteria example:
[
{"name": "myVMSSInstance1"},
{
"name": "myVMSSInstance2",
"instanceId": "2"
}
{"instanceId": "3"},
]
If the instances include two items. One with name = myVMSSInstance4
and instanceId = 2. The other with name = myVMSSInstance2 and
instanceId = 3. The criteria {"instanceId": "3"} will be the first
match since both the name and the instanceId did not match on the
first criteria.
"""
logger.debug(
"Start stop_vmss: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = choose_vmss_at_random(filter, configuration, secrets)
if not instance_criteria:
vmss_instance = choose_vmss_instance_at_random(
vmss, configuration, secrets)
else:
vmss_instance = choose_vmss_instance(
vmss, configuration, instance_criteria, secrets)
logger.debug(
"Stopping instance: {}".format(vmss_instance['name']))
client = init_client(secrets, configuration)
client.virtual_machine_scale_set_vms.power_off(
vmss['resourceGroup'],
vmss['name'],
vmss_instance['instanceId'])
def test_init_client_usgov(cmclient, spcred):
client = MagicMock()
cmclient.return_value = client
cred = MagicMock()
spcred.return_value = cred
init_client(SECRETS_USGOV, CONFIG)
spcred.assert_called_with(client_id=SECRETS_USGOV.get("client_id"),
secret=SECRETS_USGOV.get('client_secret'),
tenant=SECRETS_USGOV.get('tenant_id'),
cloud_environment=__get_cloud_env_by_name(
SECRETS_USGOV.get('azure_cloud')))
cmclient.assert_called_with(
credentials=ANY,
subscription_id=CONFIG['azure']['subscription_id'],
base_url=__get_cloud_env_by_name(
SECRETS_USGOV.get("azure_cloud")).endpoints.resource_manager)
def fetch_vmss_instances(choice, configuration, secrets):
vmss_instances = []
client = init_client(secrets, configuration)
pages = client.virtual_machine_scale_set_vms.list(choice['resourceGroup'],
choice['name'])
first_page = pages.advance_page()
vmss_instances.extend(list(first_page))
while True:
try:
page = pages.advance_page()
vmss_instances.extend(list(page))
except StopIteration:
break
results = __parse_vmss_instances_result(vmss_instances)
return results
def run(resource_group: str,
name: str,
machine: dict,
timeout: int,
parameters: dict,
secrets,
configuration,
error_msg: str = ''):
client = init_client(secrets, configuration)
poller = client.virtual_machine_scale_set_vms.run_command(
resource_group, name, machine['instanceId'], parameters)
result = poller.result(timeout) # Blocking till executed
if result and result.value:
logger.debug(result.value[0].message) # stdout/stderr
else:
raise FailedActivity(
"Operation did not finish properly. {}".format(error_msg))
def delete_vmss(filter: str = None,
instance_criteria: Iterable[Mapping[str, any]] = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Delete a virtual machine scale set instance at random.
**Be aware**: Deleting a VMSS instance is an invasive action. You will not
be able to recover the VMSS instance once you deleted it.
Parameters
----------
filter : str
Filter the virtual machine scale set. If the filter is omitted all
virtual machine scale sets in the subscription will be selected as
potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
"""
logger.debug(
"Starting delete_vmss: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = fetch_vmss(filter, configuration, secrets)
vmss_records = Records()
for scale_set in vmss:
instances_records = Records()
instances = fetch_instances(scale_set, instance_criteria,
configuration, secrets)
for instance in instances:
logger.debug("Deleting instance: {}".format(instance['name']))
client = init_client(secrets, configuration)
client.virtual_machine_scale_set_vms.delete(
scale_set['resourceGroup'], scale_set['name'],
instance['instance_id'])
instances_records.add(cleanse.vmss_instance(instance))
scale_set['virtualMachines'] = instances_records.output()
vmss_records.add(cleanse.vmss(scale_set))
return vmss_records.output_as_dict('resources')
def start_webapp(filter: str = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Start a web app at random.
Parameters
----------
filter : str
Filter the web apps. If the filter is omitted all web apps in
the subscription will be selected as potential chaos candidates.
Filtering example:
'where resourceGroup=="myresourcegroup" and name="myresourcename"'
"""
logger.debug("Start start_webapp: configuration='{}', filter='{}'".format(
configuration, filter))
choice = __fetch_webapp_at_random(filter, configuration, secrets)
logger.debug("Starting web app: {}".format(choice['name']))
client = init_client(secrets, configuration)
client.web_apps.start(choice['resourceGroup'], choice['name'])
def stress_vmss_instance_cpu(filter: str = None,
duration: int = 120,
timeout: int = 60,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Stress CPU up to 100% at random machines.
Parameters
----------
filter : str, optional
Filter the virtual machines. If the filter is omitted all machines in
the subscription will be selected as potential chaos candidates.
duration : int, optional
Duration of the stress test (in seconds) that generates high CPU usage.
Defaults to 120 seconds.
timeout : int
Additional wait time (in seconds) for stress operation to be completed.
Getting and sending data from/to Azure may take some time so it's not
recommended to set this value to less than 30s. Defaults to 60 seconds.
Examples
--------
Some calling examples. Deep dive into the filter syntax:
https://docs.microsoft.com/en-us/azure/kusto/query/
>>> stress_vmss_instance_cpu("where resourceGroup=='rg'", configuration=c,
secrets=s)
Stress all machines from the group 'rg'
>>> stress_vmss_instance_cpu("where resourceGroup=='rg' and name='name'",
configuration=c, secrets=s)
Stress the machine from the group 'rg' having the name 'name'
>>> stress_vmss_instance_cpu("where resourceGroup=='rg' | sample 2",
configuration=c, secrets=s)
Stress two machines at random from the group 'rg'
"""
logger.debug("Start stress_cpu: configuration='{}', filter='{}'".format(
configuration, filter))
vmss = choose_vmss_at_random(filter, configuration, secrets)
m = choose_vmss_instance_at_random(vmss, configuration, secrets)
name = m['name']
group = vmss['resourceGroup']
os_type = __get_os_type(m)
if os_type == OS_WINDOWS:
command_id = 'RunPowerShellScript'
script_name = "cpu_stress_test.ps1"
elif os_type == OS_LINUX:
command_id = 'RunShellScript'
script_name = "cpu_stress_test.sh"
else:
raise FailedActivity("Cannot run CPU stress test on OS: %s" % os_type)
with open(
os.path.join(os.path.dirname(__file__), "../scripts",
script_name)) as file:
script_content = file.read()
parameters = {
'command_id': command_id,
'script': [script_content],
'parameters': [{
'name': "duration",
'value': duration
}]
}
logger.debug("Stressing instance: {}".format(m['name']))
client = init_client(secrets, configuration)
poller = client.virtual_machine_scale_set_vms.run_command(
vmss['resourceGroup'], vmss['name'], m['instanceId'], parameters)
result = poller.result(duration + timeout) # Blocking till executed
if result:
logger.debug(result.value[0].message) # stdout/stderr
else:
raise FailedActivity(
"stress_vmss_instance_cpu operation did not finish on time. "
"You may consider increasing timeout setting.")
def __compute_mgmt_client(secrets, configuration):
return init_client(secrets, configuration)
