def auth(configuration: Configuration, secrets: Secrets) -> Dict[str, str]:
"""
Authenticate with the Cloud Foundry API endpoint.
The `configuration` mapping must include `"api_url"` key associated to the
URL of the API server, for example: `"https://api.local.pcfdev.io"`.
When testing against a secured endpoint exposing self-signed certificate,
you should set `"verify_ssl"` to `True`.
The `secrets` mapping must contain:
`"username"`: the user to authenticate with
`"password"`: the user's password
`"client_id"` the client id to authenticate with, defaults to `"cf"`
`"client_secret"`: the client's secret, defaults to `""`
Returns a mapping with the `access_token` and `refresh_token` keys as per
http://docs.cloudfoundry.org/api/uaa/version/4.8.0/index.html#password-grant
"""
api_url = configuration.get("cf_api_url")
verify_ssl = configuration.get("cf_verify_ssl", True)
username = secrets.get("cf_username")
password = secrets.get("cf_password")
client_id = secrets.get("cf_client_id", "cf")
client_secret = secrets.get("cf_client_secret", "")
logger.debug(
"Querying a new access token for client '{c}'".format(c=client_id))
return get_tokens(api_url, username, password, client_id, client_secret,
verify_ssl)
def rmq_client_connect(configuration: Configuration = None,
secrets: Secrets = None):
rmq_api_rest_endpoint = os.getenv(secrets.get("rabbitmq_restendpoint"))
rmq_username = os.getenv(secrets.get("rabbitmq_username"))
rmq_password = os.getenv(secrets.get("rabbitmq_password"))
return Client(rmq_api_rest_endpoint,
rmq_username,
rmq_password,
scheme='https')
def probe_app_can_connect_and_send_message_to_rabbit(
configuration: Configuration = None, secrets: Secrets = None):
if not secrets:
raise ActivityFailed(
"Please set the secrets entry to specify the SSH client settings")
# Create ssh client
client = ssh.connect(secrets)
# Get RabbitMQ connection params from secrets
rmq_api_rest_endpoint = os.getenv(secrets.get("rabbitmq_restendpoint"))
rmq_username = os.getenv(secrets.get("rabbitmq_username"))
rmq_password = os.getenv(secrets.get("rabbitmq_password"))
rmq_vhost_url = os.getenv(
secrets.get("rabbitmq_host")) + "/api/healthchecks/node"
rabbit_creds = rmq_username + ":" + rmq_password
wait_time = int(os.getenv('WAIT_TIME'))
if wait_time > 0:
logger.info("Waiting for " + str(wait_time) +
" secs before connecting to rabbit")
time.sleep(wait_time)
rabbitmq_connect_curl = 'curl -s -u {rabbitmq_creds} {rabbitmq_host}'.format(
rabbitmq_creds=rabbit_creds, rabbitmq_host=rmq_vhost_url)
stdin, stdout, stderr = client.exec_command(rabbitmq_connect_curl)
o, r = stdout.read().decode('utf-8'), stderr.read().decode('utf-8')
client.close()
logger.info(o)
error = False
if r != "":
logger.warn(r)
error = True
rmq_client = rmq_client_connect(configuration, secrets)
rmq_client.create_vhost("ce_vhost")
rmq_client.create_exchange("ce_vhost", "ce_exc", "direct")
rmq_client.create_queue("ce_vhost", "ce_que")
rmq_client.create_binding("ce_vhost", "ce_exc", "ce_que", "ce.rtkey")
if not rmq_client.publish('ce_vhost', 'ce_exc', 'ce.rtkey',
'chaos experiment message'):
error = True
rmq_client.delete_vhost("ce_vhost")
return not error
def run_python_activity(activity: Activity, configuration: Configuration,
secrets: Secrets) -> Any:
"""
Run a Python activity.
A python activity is a function from any importable module. The result
of that function is returned as the activity's output.
This should be considered as a private function.
"""
provider = activity["provider"]
mod_path = provider["module"]
func_name = provider["func"]
mod = importlib.import_module(mod_path)
func = getattr(mod, func_name)
arguments = provider.get("arguments", {}).copy()
if configuration or secrets:
arguments = substitute(arguments, configuration, secrets)
sig = inspect.signature(func)
if "secrets" in provider and "secrets" in sig.parameters:
arguments["secrets"] = {}
for s in provider["secrets"]:
arguments["secrets"].update(secrets.get(s, {}).copy())
if "configuration" in sig.parameters:
arguments["configuration"] = configuration.copy()
try:
return func(**arguments)
except Exception as x:
raise FailedActivity(
traceback.format_exception_only(
type(x), x)[0].strip()).with_traceback(sys.exc_info()[2])
def substitute(data: Union[None, str, Dict[str, Any],
List], configuration: Configuration,
secrets: Secrets) -> Dict[str, Any]:
"""
Replace forms such as `${name}` with the first value found in either the
`configuration` or `secrets` mappings within the given `data`.
The original payload is not altered.
The substitition is done recursively and inside sequences as well.
The goal is to inject values into the experiment by reading them from
dynamic values.
"""
if not data:
return data
# secrets is a mapping of mapping, only the second level is useful here
secrets = secrets.values() if secrets else []
# let's pretend we have a single mapping of everything with the config
# by the leader
mapping = ChainMap(configuration or {}, *secrets)
if isinstance(data, dict):
return substitute_dict(data, mapping)
if isinstance(data, str):
return substitute_string(data, mapping)
if isinstance(data, list):
return substitute_in_sequence(data, mapping)
return data
def get_all_events(from_time: str, to_time: str, configuration: Configuration,
secrets: Secrets) -> InstanaResponse:
"""
Get all events from instana within a time window, given by the from_time
and the to_time for details of the api see
https://instana.github.io/openapi/#tag/Events
"""
logger.debug("get_all_events")
instana_host = configuration.get("instana_host")
instana_api_token = secrets.get("instana_api_token")
if not (instana_host and instana_api_token):
raise ActivityFailed(
"No Instana Host or API Token Secrete were found.")
url = "{}/api/events".format(configuration.get("instana_host"))
params = {}
if from_time:
params["from"] = from_time
if to_time:
params["to"] = to_time
result = execute_instana_get_request(url, params, secrets)
return result
def terminate_instance_by_tag(tag_name: str = "not_set",
configuration: Configuration = None,
secrets: Secrets = None):
"""
Terminates instance marked with specified tag in aws
:param tag_name: tag to filter aws instances
:param configuration: configuration: injected by chaostoolkit framework
:return: result of modify_attribute call
"""
retval = None
ec2 = create_aws_resource(secrets, 'ec2')
filters_to_set = get_aws_filters_from_configuration(configuration)
filters_to_set.append({'Name': 'tag:' + tag_name, 'Values': [tag_name]})
dc = secrets.get("KUBERNETES_CONTEXT", "undefined")
filters_to_set.append({
'Name': 'tag:KubernetesCluster',
'Values': ["{}.k8s.wixprod.net".format(dc)]
})
filters_to_set.append({
'Name': 'instance-state-name',
'Values': ['running']
})
response = ec2.instances.filter(Filters=filters_to_set)
instances = list(response)
if len(instances) > 0:
instance = instances[0]
logger.warning('Terminate instance {} {}'.format(
instance.id, instance.private_dns_name))
retval = instance.terminate()
else:
logger.info("No aws instances found with tag {}".format(tag_name))
return retval
def vsphere_client(configuration: Configuration = None,
secrets: Secrets = None):
"""
Private function that authorizes against the vSphere API.
"""
# now setup your connection properties
host = configuration.get("vsphere_server")
port = configuration.get("vsphere_port", 443)
verify_ssl = configuration.get("vsphere_verify_ssl", True)
if secrets:
username = secrets.get("vsphere_username")
password = secrets.get("vsphere_password")
else:
username = os.getenv("VSPHERE_USERNAME")
password = os.getenv("VSPHERE_PASSWORD")
# now connect to your vCenter/vSphere
context = None
if hasattr(ssl, '_create_unverified_context'):
context = ssl._create_unverified_context()
connection = SmartConnect(host=host,
user=username,
pwd=password,
port=port,
sslContext=context)
if not connection:
print("Could not connect to the specified host using specified "
"username and password")
return connection
def apply_python_control(level: str, control: Control, experiment: Experiment,
context: Union[Activity, Experiment],
state: Union[Journal, Run, List[Run]] = None,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Apply a control by calling a function matching the given level.
"""
provider = control["provider"]
func_name = _level_mapping.get(level)
func = load_func(control, func_name)
if not func:
return
arguments = deepcopy(provider.get("arguments", {}))
if configuration or secrets:
arguments = substitute(arguments, configuration, secrets)
sig = inspect.signature(func)
if "secrets" in provider and "secrets" in sig.parameters:
arguments["secrets"] = {}
for s in provider["secrets"]:
arguments["secrets"].update(secrets.get(s, {}).copy())
if "configuration" in sig.parameters:
arguments["configuration"] = configuration.copy()
if "state" in sig.parameters:
arguments["state"] = state
if "experiment" in sig.parameters:
arguments["experiment"] = experiment
func(context=context, **arguments)
def init_resource_graph_client(secrets: Secrets) -> ResourceGraphClient:
with auth(secrets) as cred:
base_url = __get_cloud_env_by_name(
secrets.get("azure_cloud")).endpoints.resource_manager
client = ResourceGraphClient(credentials=cred, base_url=base_url)
return client
def auth(secrets: Secrets) -> ServicePrincipalCredentials:
"""
Attempt to load the Azure authentication information from a local
configuration file or the passed `configuration` mapping. The latter takes
precedence over the local configuration file.
If you provide a secrets dictionary, the returned mapping will
be created from their content. For instance, you could have:
Secrets mapping (in your experiment file):
```json
{
"azure": {
"client_id": "AZURE_CLIENT_ID",
"client_secret": "AZURE_CLIENT_SECRET",
"tenant_id": "AZURE_TENANT_ID"
}
}
```
The client_id, tenant_id, and client_secret content will be read
from the specified local environment variables, e.g. `AZURE_CLIENT_ID`,
`AZURE_TENANT_ID`, and `AZURE_CLIENT_SECRET` that you will have populated
before hand.
```
Using this function goes as follows:
```python
with auth(secrets) as cred:
azure_subscription_id = configuration['azure']['subscription_id']
resource_client = ResourceManagementClient(cred, azure_subscription_id)
compute_client = ComputeManagementClient(cred, azure_subscription_id)
```
"""
creds = dict(azure_client_id=None,
azure_client_secret=None,
azure_tenant_id=None)
if secrets:
creds["azure_client_id"] = secrets.get("client_id")
creds["azure_client_secret"] = secrets.get("client_secret")
creds["azure_tenant_id"] = secrets.get("tenant_id")
credentials = __get_credentials(creds)
yield credentials
def get_credentials(secrets: Secrets = None) -> Dict[str, str]:
"""
Credentialss may be provided via the secrets object. When they aren't,
they will be loaded from the process environment (for instance, read from
`~/.aws/credentials`).
See: https://boto3.readthedocs.io/en/latest/guide/configuration.html#guide-configuration
""" # noqa: E501
creds = dict(
aws_access_key_id=None, aws_secret_access_key=None,
aws_session_token=None)
if secrets:
creds["aws_access_key_id"] = secrets.get("aws_access_key_id")
creds["aws_secret_access_key"] = secrets.get("aws_secret_access_key")
creds["aws_session_token"] = secrets.get("aws_session_token")
return creds
def connect(secrets: Secrets = None, ):
load_dotenv()
if not secrets:
raise ActivityFailed(
"Please set the secrets entry to specify the SSH client settings")
ssh_remote_addr = os.getenv(secrets.get("remote_addr"))
username = os.getenv(secrets.get("username"))
pwd = os.getenv(secrets.get("password"))
port = os.getenv(secrets.get("port"))
client = SSHClient()
client.load_system_host_keys()
if pwd == "":
client.connect(ssh_remote_addr)
else :
client.connect(ssh_remote_addr, username=username, password=pwd, port=port)
return client
def call_api(path: str,
configuration: Configuration,
secrets: Secrets,
query: Dict[str, Any] = None,
body: Dict[str, Any] = None,
method: str = "GET",
headers: Dict[str, str] = None) -> requests.Response:
"""
Perform a Cloud Foundry API call and return the full response to the
caller.
"""
if "cf_access_token" not in secrets:
tokens = auth(configuration, secrets)
else:
tokens = {
"token_type": secrets.get("cf_token_type", "bearer"),
"access_token": secrets.get("cf_access_token")
}
h = {
"Accept":
"application/json",
"Authorization":
"{a} {t}".format(a=tokens["token_type"], t=tokens["access_token"])
}
if headers:
h.update(h)
verify_ssl = configuration.get("cf_verify_ssl", True)
url = "{u}{p}".format(u=configuration["cf_api_url"], p=path)
r = requests.request(method,
url,
params=query,
json=body,
verify=verify_ssl,
headers=h)
request_id = r.headers.get("X-VCAP-Request-ID")
logger.debug("Request ID: {i}".format(i=request_id))
if r.status_code > 399:
raise FailedActivity("failed to call '{u}': {c} => {s}".format(
u=url, c=r.status_code, s=r.text))
return r
def aws_client(resource_name: str,
configuration: Configuration = None,
secrets: Secrets = None):
"""
Create a boto3 client.
"""
configuration = configuration or {}
region = configuration.get("aws_region", "us-east-1")
creds = dict(aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None)
if secrets:
creds["aws_access_key_id"] = secrets.get("aws_access_key_id")
creds["aws_secret_access_key"] = secrets.get("aws_secret_access_key")
creds["aws_session_token"] = secrets.get("aws_session_token")
return boto3.client(resource_name, region_name=region, **creds)
def init_client(secrets: Secrets, configuration: Configuration) \
-> ComputeManagementClient:
with auth(secrets) as cred:
subscription_id = configuration['azure']['subscription_id']
base_url = __get_cloud_env_by_name(
secrets.get("azure_cloud")).endpoints.resource_manager
client = ComputeManagementClient(credentials=cred,
subscription_id=subscription_id,
base_url=base_url)
return client
def configure_control(secrets: Secrets):
"""
Enable logging to Humio if the `token` and `dataspace` are set in the
`secrets` payload.
"""
token = secrets.get("humio", {}).get("token", "").strip()
if not token:
logger.debug("Missing Humio token secret")
with_logging.enabled = False
return
logger.debug("Humio logging control is active for this session")
with_logging.enabled = True
def gandi_client(configuration: Configuration,
secrets: Secrets) -> requests.Session:
secrets = secrets or {}
api_key = secrets.get("apikey")
if not api_key:
raise ActivityFailed(
"You must provide the Gandi API key in the experiment secrets")
headers = {
"accept": "application/json",
"content-type": "application/json; charset=utf-8",
"Authorization": "Apikey {}".format(api_key.strip())
}
with requests.Session() as session:
session.headers.update(headers)
yield session
def get_event(event_id: str, configuration: Configuration,
secrets: Secrets) -> InstanaResponse:
"""
Get all an event from instana with the privded event_id for details of
the api see https://instana.github.io/openapi/#operation/getEvent
"""
logger.debug("get_event")
instana_host = configuration.get("instana_host")
instana_api_token = secrets.get("instana_api_token")
if not (instana_host and instana_api_token):
raise ActivityFailed(
"No Instana Host or API Token Secrete were found.")
url = "{}/api/events/{}".format(configuration.get("instana_host"),
event_id)
params = {}
result = execute_instana_get_request(url, params, secrets)
return result
def execute_instana_get_request(url: str, params: Dict,
secrets: Secrets) -> InstanaResponse:
"""
Call the instana rest api using the url amd params provided, use an
Authorization header from the secrets provded
"""
logger.debug("execute_instana_get_request")
logger.debug("url is : {}".format(url))
logger.debug("params : {}".format(params))
r = requests.get(url=url,
params=params,
headers={
"Authorization":
"apiToken {}".format(secrets.get("instana_api_token"))
})
if r.status_code > 399:
raise ActivityFailed("failed to call '{u}': {c} => {s}".format(
u=url, c=r.status_code, s=r.text))
logger.debug("Instana response status code: {}".format(r.status_code))
logger.debug("Instana json response: {}".format(r.json()))
return r.json()
def load_secrets(experiment_secrets: Secrets):
"""Load secrets from experiments or azure credential file.
:param experiment_secrets: Secrets provided in experiment file
:returns: a secret object
Load secrets from multiple sources that can contain different format
such as azure credential file or experiment secrets section.
The latter takes precedence over azure credential file.
Function returns following dictionary object:
```python
{
# always available
"cloud": "variable contains msrest cloud object"
# optional - available if user authenticate with service principal
"client_id": "variable contains client id",
"client_secret": "variable contains client secret",
"tenant_id": "variable contains tenant id",
# optional - available if user authenticate with existing token
"access_token": "variable contains access token",
}
```
:Loading secrets from experiment file:
Function will try to load following secrets from the experiment file:
```json
{
"azure": {
"client_id": "AZURE_CLIENT_ID",
"client_secret": "AZURE_CLIENT_SECRET",
"tenant_id": "AZURE_TENANT_ID",
"access_token": "AZURE_ACCESS_TOKEN"
}
}
```
:Loading secrets from azure credential file:
If experiment file contains no secrets, function will try to load secrets
from the azure credential file. Path to the file should be set under
AZURE_AUTH_LOCATION environment variable.
Function will try to load following secrets from azure credential file:
```json
{
"clientId": "AZURE_CLIENT_ID",
"clientSecret": "AZURE_CLIENT_SECRET",
"tenantId": "AZURE_TENANT_ID",
"resourceManagerEndpointUrl": "AZURE_RESOURCE_MANAGER_ENDPOINT",
...
}
```
More info about azure credential file may be found:
https://docs.microsoft.com/en-us/azure/developer/python/azure-sdk-authenticate
"""
# 1: lookup for secrets in experiment file
if experiment_secrets:
return {
'client_id': experiment_secrets.get('client_id'),
'client_secret': experiment_secrets.get('client_secret'),
'tenant_id': experiment_secrets.get('tenant_id'),
# load cloud object
'cloud': cloud.get_or_raise(experiment_secrets.get('azure_cloud')),
'access_token': experiment_secrets.get('access_token'),
}
# 2: lookup for credentials in azure auth file
az_auth_file = _load_azure_auth_file()
if az_auth_file:
rm_endpoint = az_auth_file.get('resourceManagerEndpointUrl')
return {
'client_id': az_auth_file.get('clientId'),
'client_secret': az_auth_file.get('clientSecret'),
'tenant_id': az_auth_file.get('tenantId'),
# load cloud object
'cloud': azure_cloud.get_cloud_from_metadata_endpoint(rm_endpoint),
# access token is not supported for credential files
'access_token': None,
}
# no secretes
logger.warn("Unable to load Azure credentials.")
return {}
def auth(secrets: Secrets) -> ServicePrincipalCredentials:
"""
Attempt to load the Azure authentication information from a local
configuration file or the passed `configuration` mapping. The latter takes
precedence over the local configuration file. Service principle and token
based credentials are supported. Token based credentials do not currently
support refresh token functionality.
If you provide a secrets dictionary, the returned mapping will
be created from their content. For instance, for service principle based
credentials, you could have:
Secrets mapping (in your experiment file):
```json
{
"azure": {
"client_id": "AZURE_CLIENT_ID",
"client_secret": "AZURE_CLIENT_SECRET",
"tenant_id": "AZURE_TENANT_ID"
}
}
```
Also if you are not working with Public Global Azure, e.g. China Cloud
You can feed the cloud environment name as well.
Please refer to msrestazure.azure_cloud
```json
{
"azure": {
"client_id": "xxxxxxx",
"client_secret": "*******",
"tenant_id": "@@@@@@@@@@@",
"azure_cloud": "AZURE_CHINA_CLOUD"
}
}
```
The client_id, tenant_id, and client_secret content will be read
from the specified local environment variables, e.g. `AZURE_CLIENT_ID`,
`AZURE_TENANT_ID`, and `AZURE_CLIENT_SECRET` that you will have populated
before hand.
If the client_secret is not provided, then token based credentials is
assumed and an access_token value must be present and updated as the token
expires.
```
Using this function goes as follows:
```python
with auth(secrets) as cred:
azure_subscription_id = configuration['azure']['subscription_id']
resource_client = ResourceManagementClient(cred, azure_subscription_id)
compute_client = ComputeManagementClient(cred, azure_subscription_id)
```
Again, if you are not working with Public Azure Cloud,
and you set azure_cloud in secret,
this will pass one more parameter `base_url` to above function.
```python
cloud = __get_cloud_env_by_name(creds['azure_cloud'])
client = ComputeManagementClient(
credentials=cred, subscription_id=subscription_id,
base_url=cloud.endpoints.resource_manager)
```
"""
creds = dict(azure_client_id=None,
azure_client_secret=None,
azure_tenant_id=None,
azure_cloud=None,
access_token=None)
if secrets:
creds["azure_client_id"] = secrets.get("client_id")
creds["azure_client_secret"] = secrets.get("client_secret")
creds["azure_tenant_id"] = secrets.get("tenant_id")
creds["azure_cloud"] = secrets.get("azure_cloud", "AZURE_PUBLIC_CLOUD")
creds["access_token"] = secrets.get("access_token")
credentials = __get_credentials(creds)
yield credentials
def client(service_name: str,
version: str = 'v1',
secrets: Secrets = None) -> Resource:
"""
Create a client for the given service.
To authenticate, you need to create a service account manually and either
pass the filename or the content of the file into the `secrets` object.
So, in the experiment, use one of the followings:
```json
{
"gce": {
"service_account_file": "/path/to/file.json"
}
}
```
```json
{
"gce": {
"service_account_info": {
"type": "service_account",
"project_id": "...",
"private_key_id": "...",
"private_key": "...",
"client_email": "...",
"client_id": "...",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/...."
}
}
}
```
You would likely want to read value from the environment or Vault if you
use the second approach, and avoid storing sensitive data into the
experiment itself.
Make sure your service account has enough permissions for the activities
you wish to conduct (though do not give it too wide permissions either).
See: https://developers.google.com/api-client-library/python/auth/service-accounts
Also: http://google-auth.readthedocs.io/en/latest/reference/google.oauth2.service_account.html
""" # noqa: E501
secrets = secrets or {}
service_account_file = secrets.get("service_account_file")
service_account_info = secrets.get("service_account_info")
credentials = None
if service_account_file:
service_account_file = os.path.expanduser(service_account_file)
if not os.path.exists(service_account_file):
raise FailedActivity("GCE account settings not found at {}".format(
service_account_file))
logger.debug(
"Using GCE credentials from file: {}".format(service_account_file))
credentials = Credentials.from_service_account_file(
service_account_file)
elif service_account_info and isinstance(service_account_info, dict):
logger.debug("Using GCE credentials embedded into secrets")
credentials = Credentials.from_service_account_info(
service_account_info)
else:
raise FailedActivity(
"missing GCE credentials settings in secrets of this activity")
if credentials is not None and credentials.expired:
logger.debug("GCE credentials need to be refreshed as they expired")
credentials.refresh(httplib2.Http())
if not credentials:
raise FailedActivity(
"missing a service account to authenticate with the "
"Google Cloud Services")
return build(service_name, version=version, credentials=credentials)
def get_client(secrets: Secrets) -> WebClient:
slack_token = secrets.get("slack", {}).get("token")
client = WebClient(token=slack_token)
return client
