def bosh_login(*, user=None, pswd=None, cacert=None, env=None):
"""
Login to the bosh CLI. All parameters default to config values if not specified.
:param user: Bosh username.
:param pswd: Bosh password.
:param cacert: Bosh client cert.
:param env: Bosh environment to login to.
:return: int, str, str; Returncode, stdout, stderr.
"""
cfg = Config()['bosh']
certfile = Certfile(cacert=cacert)
creds = cfg.get('credentials')
if not creds:
assert user and pswd, "Must specify username and password!"
else:
user = user or creds['user']
pswd = pswd or creds['pswd']
cmd = [cfg['cmd'], '-e', env or cfg['env']]
stdin = [user, pswd]
if certfile:
cmd.extend(['--ca-cert=' + certfile.name])
cmd.append('login')
return run_cmd(cmd, stdin)
def cf_login(*, user=None, pswd=None, api=None, skip_ssl_validation=None):
"""
Login to the CF CLI. All parameters default to config values if not specified.
:param user: str; Cf username.
:param pswd: str; Cf password.
:param api: str; Cf api url.
:param skip_ssl_validation: bool; Whether cf ssl validation should be skipped, useful if certs are not installed.
:return: int, str, str; Returncode, stdout, stderr.
"""
cfg = Config()['cf']
creds = cfg.get('credentials')
if not creds:
assert user and pswd and api, "Must specify username, password, and API address!"
else:
user = user or creds['user']
pswd = pswd or creds['pswd']
if skip_ssl_validation is None:
skip_ssl_validation = creds.get('skip-ssl-validation') or False
pswd = '"{}"'.format(pswd.replace('"', r'\"'))
api = api or creds['api']
cmd = [cfg['cmd'], 'login']
if skip_ssl_validation:
cmd.append('--skip-ssl-validation')
cmd.extend(['-a', api, '-u', user, '-p', pswd])
return run_cmd(cmd, stdin='\n')
def find_application_routes(app_guid):
"""
Find the url routes of an application.
:param app_guid: The application GUID for which to find routes.
:return List[str]: The app's route(s).
"""
cfg = Config()
routes = []
rcode, stdout, _ = util.run_cmd(
[cfg['cf']['cmd'], 'curl', '/v2/apps/{}/routes'.format(app_guid)])
if rcode:
sys.exit("Failure to call cf curl!")
resources = map(lambda v: v['entity'],
util.extract_json(stdout)[0]['resources'])
for resource in resources:
host = resource['host']
path = resource['path']
rcode, stdout, _ = util.run_cmd(
[cfg['cf']['cmd'], 'curl', resource['domain_url']])
if rcode:
sys.exit("Failure to call cf curl!")
domain = util.extract_json(stdout)[0]['entity']['name']
route = '{}.{}'.format(host, domain)
if path and path != '':
route += '/{}'.format(path)
routes.append(route)
return routes
def bosh_cli(args, stdin=None, env=None, dep=None, suppress_output=False):
"""
Call the bosh CLI.
:param args: Union[List[str], str]; Arguments for the bosh CLI. This will allow chaining additional commands
with '&&', '|', etc.
:param env: str; The bosh environment to use. Defaults to config value.
:param dep: str; The bosh deployment to use. Defaults to configured cf deployment value.
:param stdin: Optional[Union[str, List[Union[str, List[str]]]]]; Input to pipe to the program.
:param suppress_output: bool; If true, no extra debug output will be printed when an error occurs.
:return: int, str, str; Returncode, stdout, stderr.
"""
boshcfg = Config()['bosh']
certfile = Certfile()
cmd = [
boshcfg['cmd'], '-e', env or boshcfg['env'], '-d', dep
or boshcfg['cf-dep']
]
if certfile:
cmd.extend(['--ca-cert=' + certfile.name])
if isinstance(args, list):
cmd.extend(args)
else:
cmd.append(args)
res = run_cmd(cmd, stdin=stdin, suppress_output=suppress_output)
return res
def url():
cfg = Config()
depcfg = cfg['testing']
push_app = bool(depcfg.get('push-app'))
if push_app: # got to push an app before we get the URL
start_dir = os.getcwd()
with TemporaryDirectory() as td:
print("Downloading Spring Music")
assert_cmd(['git clone https://github.com/cloudfoundry-samples/spring-music.git', td], timeout=60*2)
os.chdir(td)
assert_cmd('git checkout 963d307f9c0da020545d13c947ad6d5472e29c94')
print("Building Spring Music")
assert_cmd('./gradlew clean assemble', timeout=60*5)
print("Deploying Spring Music...")
assert_cmd([cfg['cf']['cmd'], 'create-service', depcfg['db-market-name'], depcfg['db-plan'],
depcfg['db-instance-name']])
assert_cmd([cfg['cf']['cmd'], 'push', depcfg['appname']], timeout=60*5)
assert_cmd([cfg['cf']['cmd'], 'bind-service', depcfg['appname'], depcfg['db-instance-name']])
assert_cmd([cfg['cf']['cmd'], 'restage', depcfg['appname']], timeout=60*5)
os.chdir(start_dir)
routes = find_application_routes(find_application_guid(depcfg['appname']))
url = routes[0]
yield url
if push_app:
print("Tearing down Spring Music...")
assert_cmd([cfg['cf']['cmd'], 'delete -f -r', depcfg['appname']])
assert_cmd([cfg['cf']['cmd'], 'delete-service -f', depcfg['db-instance-name']])
def run_ctk(func, cfg, org, space, appname, msg=None):
"""
This is a helper function to reduce code duplication when called by Chaos Toolkit.
:param func: Fn[app] -> Optional[any]; A function which performs some actions using an app object.
:param cfg: Dict[String, any]; Configuration information about the environment.
:param org: String; Cloud Foundry organization containing the application.
:param space: String; Cloud Foundry space containing the application.
:param appname: String; Application in Cloud Foundry which is to be targeted.
:param msg: Optional[String]; A message to display at the beginning of operations.
:return: Dict[String, Any]; The serialized App object after all operations were performed.
"""
if cfg:
Config().load_dict(cfg)
if msg:
logger.info(msg)
try:
app = App.discover(org, space, appname)
if not app:
raise FailedActivity("Error discovering app!")
result = func(app)
except SystemExit as err:
logger.exception(err)
raise FailedActivity(err)
if result:
raise FailedActivity(
"Error performing operation! Function returned {}.".format(result))
logger.info("Done!")
return app.serialize()
def __init__(self, cacert=None):
"""
Get the certfile from the config if it exists, otherwise just init to None.
:param cacert: str; Cert override (instead of reading config).
"""
self._crtfile = None
self.name = None
if not cacert:
creds = Config()['bosh'].get('credentials')
if creds:
cacert = creds.get('cacert')
if cacert:
self._crtfile = NamedTemporaryFile(mode='w')
self._crtfile.write(cacert)
self._crtfile.flush()
self.name = self._crtfile.name
def cf_target(org, space):
"""
Target a specific organization and space using the cloud foundry CLI. This should be run before anything which calls
out to cloud foundry. This will fail if cloud foundry is not logged in.
:param org: The organization to target.
:param space: The space within the organization to target.
:return: The returncode of the cloud foundry CLI.
"""
cfg = Config()
rcode, _, _ = run_cmd([cfg['cf']['cmd'], 'target', '-o', org, '-s', space])
return rcode
def run_cmd_on_container(dcid, contid, cmd, suppress_output=False):
"""
Run one or more commands in the shell on a container on a diego cell.
:param dcid: str; Diego-cell ID of the Diego Cell running the container.
:param contid: str; Container ID of the container which is to be connected to.
:param cmd: Union[str, List[str]]; Command(s) to run on the container.
:param suppress_output: bool; If true, no extra debug output will be printed when an error occurs.
:return: int, str, str; Returncode, stdout, stderr.
"""
cfg = Config()
use_containerd = cfg.get("use-containerd") is not None
if use_containerd:
# Refer to: https://devops.stackexchange.com/a/13781/27344
shell_cmd = f'exec sudo /var/vcap/packages/containerd/bin/ctr -a /var/vcap/sys/run/containerd/containerd.sock -n garden tasks exec --exec-id my-shell --tty {contid} /bin/bash'
else:
shell_cmd = f'exec sudo /var/vcap/packages/runc/bin/runc exec -t {contid} /bin/bash'
if isinstance(cmd, list):
cmd.insert(0, shell_cmd)
else:
cmd = [shell_cmd, cmd]
return run_cmd_on_diego_cell(dcid, cmd, suppress_output=suppress_output)
def get_apps():
"""
Get the apps deployed in the cloud foundry cluster.
:return: Example app object:
```
{
'app_guid': '57c6b9cc-7e48-4178-a8fc-a312afcdb42a'
'process_guid': '57c6b9cc-7e48-4178-a8fc-a312afcdb42a-3d4c0341-31f1-49b3-8a49-41e100122fd6',
'index': 0,
'domain': 'cf-apps',
'instance_guid': 'b39b4d2a-f36d-4aaf-42a9-0d1f',
'cell_id': '373e05d9-270f-4358-935e-7af712078c4a',
'address': '10.94.173.17',
'ports': [
{
'container_port': 8080,
'host_port': 61028,
'container_tls_proxy_port': 61001,
'host_tls_proxy_port': 61094
},
{
'container_port': 2222,
'host_port': 61029,
'container_tls_proxy_port': 61002,
'host_tls_proxy_port': 61095
}
],
'instance_address': '198.19.84.194',
'crash_count': 0,
'state': 'RUNNING',
'since': 1551811902829311393,
'modification_tag': {
'epoch': '522dc0a9-88f9-4314-7ceb-5ee8a6402f85',
'index': 2
}
}
```
"""
cfg = Config()
rcode, stdout, _ = monarch.pcf.util.run_cmd_on_diego_cell(
cfg['bosh']['cfdot-dc'],
['source /etc/profile.d/cfdot.sh', 'cfdot actual-lrp-groups'])
if rcode:
logger.error("Failed retrieving actual LRP grups from %s",
cfg['bosh']['cfdot-dc'])
return None
apps = []
for app in util.extract_json(stdout):
app = app['instance']
app['app_guid'] = app['process_guid'][:36]
apps.append(app)
return apps
def add_services_from_cfg(self):
"""
Read any custom services defined in the config and add them.
:return: Dict[String, Service]; The list of all services bound to this application.
"""
cfg = Config()
if 'services' not in cfg:
return self.services
for service in cfg['services']:
self.add_custom_service(service['name'], service['host'],
[tuple(i) for i in service['ports']],
service['user'], service['password'])
return self.services
def block_services(self, services=None, direction='egress'):
"""
Block this application from accessing its services on all its known hosts.
:param services: List[String]; List of service names to block, will target all if unset.
:param direction: str; Traffic direction to block.
:return: int; A returncode if any of the bosh ssh instances do not return 0.
"""
cfg = Config()
direction = util.parse_direction(direction)
assert direction, "Could not parse direction!"
for app_instance in self.instances:
cmds = []
for service in self.services:
if service['type'] in cfg['service-whitelist']:
continue
if services and service['name'] not in services:
continue
logger.info("Blocking %s for %s:%s", service['name'],
app_instance['diego_id'], app_instance['cont_ip'])
for (sip, protocol, port) in service['hosts']:
if direction in {'egress', 'both'}:
cmd = [
'sudo', 'iptables', '-I', 'FORWARD', '1', '-s',
app_instance['cont_ip'], '-d', sip, '-p', protocol
]
if port != 'all':
cmd.extend(['--dport', port])
cmd.extend(['-j', 'DROP'])
cmds.append(' '.join(map(str, cmd)))
if direction in {'ingress', 'both'}:
cmd = [
'sudo', 'iptables', '-I', 'FORWARD', '1', '-d',
app_instance['cont_ip'], '-s', sip, '-p', protocol
]
if port != 'all':
cmd.extend(['--sport', port])
cmd.extend(['-j', 'DROP'])
cmds.append(' '.join(map(str, cmd)))
if not cmds:
continue
rcode, _, _ = app_instance.run_cmd_on_diego_cell(cmds)
if rcode:
logger.error("Received return code %d from iptables call.",
rcode)
self.unblock_services(services=services)
return rcode
return 0
def find_application_guid(appname):
"""
Find the GUID of an application using cloud foundry's CLI interface. The GUID acts as a unique identifier for
the application which we can then use to find what containers are running it.
:param appname: String; The name of the app to deserialize.
:return: String; The application GUID.
"""
assert appname
cfg = Config()
cmd = '{} app {} --guid'.format(cfg['cf']['cmd'], appname)
rcode, stdout, _ = util.run_cmd(cmd)
guid = stdout.splitlines()[0]
if rcode:
sys.exit(
"Failed retrieving the GUID for the specified app. Make sure {} is in this space!"
.format(appname))
logger.debug(guid)
return guid
def unblock_services(self, services=None):
"""
Unblock this application from accessing its services on all its known hosts.
:param services: List[String]; List of service names to unblock, will target all if unset.
"""
cfg = Config()
for app_instance in self.instances:
cmds = []
for service in self.services:
if service['type'] in cfg['service-whitelist']:
continue
if services and service['name'] not in services:
continue
logger.info("Unblocking %s for %s:%s", service['name'],
app_instance['diego_id'], app_instance['cont_ip'])
for (sip, protocol, port) in service['hosts']:
cmd = [
'sudo', 'iptables', '-D', 'FORWARD', '-s',
app_instance['cont_ip'], '-d', sip, '-p', protocol
]
if port != 'all':
cmd.extend(['--dport', port])
cmd.extend(['-j', 'DROP'])
for _ in range(TIMES_TO_REMOVE):
cmds.append(' '.join(map(str, cmd)))
cmd = [
'sudo', 'iptables', '-D', 'FORWARD', '-d',
app_instance['cont_ip'], '-s', sip, '-p', protocol
]
if port != 'all':
cmd.extend(['--sport', port])
cmd.extend(['-j', 'DROP'])
for _ in range(TIMES_TO_REMOVE):
cmds.append(' '.join(map(str, cmd)))
if not cmds:
continue
monarch.pcf.util.run_cmd_on_diego_cell(app_instance['diego_id'],
cmds,
suppress_output=True)
def find_application_services(appname):
"""
Discover all services bound to an application. This will use `cf env` and parse the output for VCAP_SERVICES.
:param appname: String; The name of the app to deserialize.
:return: List[Service]; The list of all services bound to this application.
"""
cfg = Config()
rcode, stdout, _ = util.run_cmd('{} env {}'.format(cfg['cf']['cmd'],
appname))
if rcode:
sys.exit("Failed to query application environment variables.")
json_objs = util.extract_json(stdout)
if not json_objs:
sys.exit("Error reading output from `cf env`")
for obj in json_objs:
if 'VCAP_SERVICES' not in obj:
json_objs.remove(obj)
if len(json_objs) != 1:
logger.info("No services found for %s.", appname)
return []
services = []
vservices = json_objs[0]['VCAP_SERVICES']
logger.debug(json.dumps(vservices, indent=' '))
for sname, sconfig in vservices.items():
for instance_cfg in sconfig:
service = Service.from_service_info(sname, instance_cfg)
if service:
logger.info("Found service: %s", service)
services.append(service)
return services
def find_application_instances(app_guid):
"""
Finds the instances of an application and extracts the relevant information.
:return: List[AppInstance]; The app instances app and their associated hosts.
"""
cfg = Config()
# for each instance, find information about where it is hosted and its connected ports
instances = []
raw_apps = bosh.get_apps()
if not raw_apps:
logger.warning("No application instances found for %s.", app_guid)
return None
for instance in raw_apps:
if instance['app_guid'] != app_guid:
continue
if instance['state'] != 'RUNNING':
continue
diego_ip = instance['address']
cont_ip = instance['instance_address']
diego_id = 'diego_cell/' + instance['cell_id']
app_ports = set(
) # ports the application is listening on within the container
for ports in instance['ports']:
diego_port = ports['host_port'] # node port on the diego-cell
cont_port = ports[
'container_port'] # port the application is listening on in the container
if diego_port in cfg['host-port-whitelist']:
continue
if cont_port in cfg['container-port-whitelist']:
continue
app_ports.add((diego_port, cont_port))
logger.debug('Found application at %s:%d with container port %d',
diego_ip, diego_port, cont_port)
# Lookup the virtual network interface
_, stdout, _ = monarch.pcf.util.run_cmd_on_diego_cell(diego_id, 'ip a')
stdout = util.group_lines_by_hanging_indent(stdout)
index = util.find_string_in_grouping(stdout,
cont_ip.replace('.', r'\.'))
if not index:
logger.warning("Could not find virtual interface!")
diego_vi = None
else:
diego_vi = stdout[index[0]][0] # want to get parent of the match
match = re.match(r'\d+: ([\w-]+)(@[\w-]+)?:', diego_vi)
assert match # This should never fail, so the regex must be wrong!
diego_vi = match[1]
logger.debug("Hosting diego-cell Virtual Interface: %s", diego_vi)
# Lookup the Container ID
cmd = "sudo cat /var/vcap/sys/log/rep/rep.stdout.log | grep {} | tail -n 1".format(
cont_ip)
rcode, stdout, _ = monarch.pcf.util.run_cmd_on_diego_cell(
diego_id, cmd)
if rcode:
logger.error("Failed retrieving container GUID from %s.", diego_id)
cont_id = None
else:
cont_id = util.extract_json(stdout)[0]['data']['container-guid']
logger.debug("Hosting container GUID: %s.", cont_id)
# Record the app instance information
app_instance = AppInstance(diego_id=diego_id,
diego_ip=diego_ip,
cont_id=cont_id,
cont_ip=cont_ip,
app_ports=app_ports,
diego_vi=diego_vi)
instances.append(app_instance)
logger.info("Found instance: %s", app_instance)
return instances
def _app_port_not_whitelisted(ports):
cfg = Config()
return ports[0] not in cfg['host-port-whitelist'] and \
ports[1] not in cfg['container-port-whitelist']
def app():
cfg = Config()['testing']
app = App.discover(cfg['org'], cfg['space'], cfg['appname'])
yield app
if app:
app.undo_all()
def cfg():
cfg = Config()
cfg.load_yaml('tests/config/app_test.yml')
yield cfg