def create(definition, region, version, parameter, disable_rollback, dry_run, force):
'''Create a new Cloud Formation stack from the given Senza definition file'''
input = definition
region = get_region(region)
check_credentials(region)
account_info = AccountArguments(region=region)
args = parse_args(input, region, version, parameter, account_info)
with Action('Generating Cloud Formation template..'):
data = evaluate(input.copy(), args, account_info, force)
cfjson = json.dumps(data, sort_keys=True, indent=4)
stack_name = "{0}-{1}".format(input["SenzaInfo"]["StackName"], version)
if len(stack_name) > 128:
fatal_error('Error: Stack name "{}" cannot exceed 128 characters. '.format(stack_name) +
'Please choose another name/version.')
parameters = []
for name, parameter in data.get("Parameters", {}).items():
parameters.append([name, getattr(args, name, None)])
tags = {}
for tag in input["SenzaInfo"].get('Tags', []):
for key, value in tag.items():
# # As the SenzaInfo is not evaluated, we explicitly evaluate the values here
tags[key] = evaluate_template(value, info, [], args, account_info)
tags.update({
"Name": stack_name,
"StackName": input["SenzaInfo"]["StackName"],
"StackVersion": version
})
if "OperatorTopicId" in input["SenzaInfo"]:
topic = input["SenzaInfo"]["OperatorTopicId"]
topic_arn = resolve_topic_arn(region, topic)
if not topic_arn:
fatal_error('Error: SNS topic "{}" does not exist'.format(topic))
topics = [topic_arn]
else:
topics = None
capabilities = get_required_capabilities(data)
cf = boto.cloudformation.connect_to_region(region)
with Action('Creating Cloud Formation stack {}..'.format(stack_name)) as act:
try:
if dry_run:
info('**DRY-RUN** {}'.format(topics))
else:
cf.create_stack(stack_name, template_body=cfjson, parameters=parameters, tags=tags,
notification_arns=topics, disable_rollback=disable_rollback, capabilities=capabilities)
except boto.exception.BotoServerError as e:
if e.error_code == 'AlreadyExistsException':
act.fatal_error('Stack {} already exists. Please choose another version.'.format(stack_name))
else:
raise
def target_update(obj, target_uri, sli_name, target_from, target_to,
target_file):
"""Update Target for a product SLO."""
client = get_client(obj)
target = client.target_get(target_uri)
if not target:
fatal_error('Target {} does not exist'.format(target_uri))
product = client.product_list(name=target['product_name'])[0]
sli = client.sli_list(product=product, name=sli_name)
if not sli:
fatal_error('SLI {} does not exist'.format(sli_name))
sli = sli[0]
with Action('Updating Target {} for product {}'.format(
target_uri, target['product_name']),
nl=True) as act:
if target_file:
target = json.load(target_file)
else:
if sli_name:
target['sli_uri'] = sli['uri']
if target_from:
target['from'] = target_from
if target_to:
target['to'] = target_to
validate_target(target, act)
if not act.errors:
target = client.target_update(target)
print(json.dumps(target, indent=4))
def delete(stack_ref: List[str],
region: str, dry_run: bool, force: bool, remote: str):
"""Delete Cloud Formation stacks"""
lizzy = setup_lizzy_client(remote)
stack_refs = get_stack_refs(stack_ref)
all_with_version = all(stack.version is not None
for stack in stack_refs)
# this is misleading but it's the current behaviour of senza
# TODO Lizzy list (stack_refs) to see if it actually matches more than one stack
# to match senza behaviour
if (not all_with_version and not dry_run and not force):
fatal_error(
'Error: {} matching stacks found. '.format(len(stack_refs)) +
'Please use the "--force" flag if you really want to delete multiple stacks.')
# TODO pass force option to agent
output = ''
for stack in stack_refs:
if stack.version is not None:
stack_id = '{stack.name}-{stack.version}'.format(stack=stack)
else:
stack_id = stack.name
with Action("Requesting stack '{stack_id}' deletion..",
stack_id=stack_id):
output = lizzy.delete(stack_id, region=region, dry_run=dry_run)
print(output)
def slo_update(obj, product_name, slo_id, title, description, slo_file):
"""
Update SLO for a product. All targets will be ignored if specified in SLO definitions.
Updating targets can be achieved via "target" command.
"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
slo = client.slo_list(product, id=slo_id)
if not slo:
fatal_error('SLO {} does not exist'.format(slo_id))
slo = slo[0]
with Action('Updating SLO {} for product {}'.format(slo_id, slo['product_name']), nl=True) as act:
if slo_file:
slo = json.load(slo_file)
slo['uri'] = slo['uri']
else:
if title:
slo['title'] = title
if description:
slo['description'] = description
validate_slo(slo, act)
if not act.errors:
slo = client.slo_update(slo)
print(json.dumps(slo, indent=4))
def slo_create(obj, product_name, title, description, slo_file):
"""
Create SLO. If SLO file is used, then --title and --description will be ignored.
"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
with Action('Creating SLO for product: {}'.format(product_name),
nl=True) as act:
if slo_file:
slo = json.load(slo_file)
else:
slo = {'title': title, 'description': description}
validate_slo(slo, act)
if not act.errors:
new_slo = client.slo_create(product, slo['title'],
slo.get('description', ''))
print(json.dumps(new_slo, indent=4))
for target in slo.get('targets', []):
t = client.target_create(new_slo,
target['sli_uri'],
target_from=target['from'],
target_to=target['to'])
act.ok('Created a new target')
print(json.dumps(t, indent=4))
def product_update(obj, name, new_name, new_email, product_group_name):
"""Update product"""
client = get_client(obj)
ps = client.product_list(name)
if not ps:
fatal_error('Product {} does not exist'.format(name))
with Action('Updating product: {}'.format(name), nl=True):
p = ps[0]
if new_name:
p['name'] = new_name
if new_email:
p['email'] = new_email
if product_group_name:
pgs = client.product_group_list(name=product_group_name)
if not pgs:
fatal_error('Product group {} does not exist'.format(
product_group_name))
p['product_group_uri'] = pgs[0]['uri']
p = client.product_update(p)
print(json.dumps(p, indent=4))
def sli_update(obj, product_name, name, sli_file):
"""Update SLI for a product"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
slis = client.sli_list(product, name)
if not slis:
fatal_error('SLI {} does not exist'.format(name))
with Action('Updating SLI {} for product: {}'.format(name, product_name),
nl=True) as act:
sli = json.load(sli_file)
validate_sli(obj, sli, act)
if not act.errors:
sli['uri'] = slis[0]['uri']
s = client.sli_update(sli)
print(json.dumps(s, indent=4))
def output(output):
'''Example for all possible Echo Formats
You see the message only, if the Output TEXT
'''
with OutputFormat(output):
action('This is a ok:')
ok()
action('This is a ok with message:')
ok('all is fine')
action('This is a warning:')
warning('please check this')
with Action('Start with working..') as act:
# save_the_world()
act.progress()
act.progress()
act.progress()
act.progress()
print_table('id name'.split(), [{
'id': 1,
'name': 'Test #1'
}, {
'id': 2,
'name': 'Test #2'
}])
info('Only FYI')
action('This is a error:')
error('this is wrong, please fix')
action('This is a fatal error:')
fatal_error('this is a fuckup')
info('I\'am not printed, the process a dead')
def output(output):
'''Example for all possible Echo Formats
You see the message only, if the Output TEXT
'''
with OutputFormat(output):
action('This is a ok:')
ok()
action('This is a ok with message:')
ok('all is fine')
action('This is a warning:')
warning('please check this')
with Action('Start with working..') as act:
# save_the_world()
act.progress()
act.progress()
act.progress()
act.progress()
print_table('id name'.split(), [{'id': 1, 'name': 'Test #1'}, {'id': 2, 'name': 'Test #2'}])
info('Only FYI')
action('This is a error:')
error('this is wrong, please fix')
action('This is a fatal error:')
fatal_error('this is a fuckup')
info('I\'am not printed, the process a dead')
def connection_error(e: requests.ConnectionError, fatal=True):
reason = e.args[0].reason # type: requests.packages.urllib3.exceptions.NewConnectionError
_, pretty_reason = str(reason).split(':', 1)
msg = ' {}'.format(pretty_reason)
if fatal:
fatal_error(msg)
else:
error(msg)
def validate_pierone_url(url: str) -> None:
ping_url = url.rstrip('/') + '/swagger.json'
try:
response = requests.get(ping_url, timeout=5)
response.raise_for_status()
if 'Pier One API' not in response.text:
fatal_error('ERROR: Did not find a valid Pier One registry at {}'.format(url))
except RequestException:
fatal_error('ERROR: Could not reach {}'.format(ping_url))
def validate_pierone_url(url: str) -> None:
ping_url = url.rstrip('/') + '/swagger.json'
try:
response = requests.get(ping_url, timeout=5)
response.raise_for_status()
if 'Pier One API' not in response.text:
fatal_error('ERROR: Did not find a valid Pier One registry at {}'.format(url))
except RequestException:
fatal_error('ERROR: Could not reach {}'.format(ping_url))
def product_group_get(obj, name):
"""List all product groups"""
client = get_client(obj)
pgs = client.product_group_list(name)
if not pgs:
fatal_error('Product group {} does not exist'.format(name))
print(json.dumps(pgs[0], indent=4))
def list_stacks(stack_ref: str, all: bool, watch: int, output: str):
"""List Lizzy stacks"""
config = Configuration()
access_token = fetch_token(config.token_url, config.scopes,
config.credentials_dir)
lizzy = Lizzy(config.lizzy_url, access_token)
repeat = True
while repeat:
try:
all_stacks = lizzy.get_stacks()
except requests.RequestException as e:
fatal_error('Failed to get stacks: {}'.format(e))
if all:
stacks = all_stacks
else:
stacks = [
stack for stack in all_stacks
if stack['status'] not in ['LIZZY:REMOVED']
]
if stack_ref:
stacks = [
stack for stack in stacks if stack['stack_name'] in stack_ref
]
rows = []
for stack in stacks:
creation_time = dateutil.parser.parse(stack['creation_time'])
rows.append({
'stack_name': stack['stack_name'],
'version': stack['stack_version'],
'image_version': stack['image_version'],
'status': stack['status'],
'creation_time': creation_time.timestamp()
})
rows.sort(key=lambda x: (x['stack_name'], x['version']))
with OutputFormat(output):
print_table(
'stack_name version image_version status creation_time'.split(
),
rows,
styles=STYLES,
titles=TITLES)
if watch: # pragma: no cover
time.sleep(watch)
click.clear()
else:
repeat = False
def product_delete(obj, name):
"""Delete a product"""
client = get_client(obj)
with Action('Deleting product: {}'.format(name), nl=True):
p = client.product_list(name=name)
if not p:
fatal_error('Product {} does not exist!'.format(name))
client.product_delete(p[0])
def product_get(obj, name):
"""Get product"""
client = get_client(obj)
p = client.product_list(name=name)
if not p:
fatal_error('Product {} does not exist'.format(name))
print(json.dumps(p[0], indent=4))
def sli_list(obj, product_name):
"""List SLIs for a product"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
res = client.sli_list(product[0])
print(json.dumps(res, indent=4))
def validate_wsgi_server_requirements(ctx, param, value):
if value == 'gevent':
try:
import gevent # NOQA
except:
fatal_error('gevent library is not installed')
elif value == 'tornado':
try:
import tornado # NOQA
except:
fatal_error('tornado library is not installed')
def validate_wsgi_server_requirements(ctx, param, value):
if value == 'gevent':
try:
import gevent # NOQA
except:
fatal_error('gevent library is not installed')
elif value == 'tornado':
try:
import tornado # NOQA
except:
fatal_error('tornado library is not installed')
def main():
try:
cli()
except requests.HTTPError as e:
detail = ''
try:
detail = e.response.json()['detail']
except:
pass
fatal_error('HTTP error: {} - {} - {}'.format(e.response.status_code, e.response.reason, detail))
def create_cf_template(definition, region, version, parameter, force):
region = get_region(region)
check_credentials(region)
account_info = AccountArguments(region=region)
args = parse_args(definition, region, version, parameter, account_info)
with Action('Generating Cloud Formation template..'):
data = evaluate(definition.copy(), args, account_info, force)
stack_name = "{0}-{1}".format(data['Mappings']['Senza']['Info']['StackName'],
data['Mappings']['Senza']['Info']['StackVersion'])
if len(stack_name) > 128:
fatal_error('Error: Stack name "{}" cannot exceed 128 characters. '.format(stack_name) +
'Please choose another name/version.')
parameters = []
for name, parameter in data.get("Parameters", {}).items():
parameters.append({'ParameterKey': name, 'ParameterValue': getattr(args, name, None)})
tags = {}
senza_tags = data['Mappings']['Senza']['Info'].get('Tags')
if isinstance(senza_tags, dict):
tags.update(senza_tags)
elif isinstance(senza_tags, list):
for tag in senza_tags:
for key, value in tag.items():
# # As the SenzaInfo is not evaluated, we explicitly evaluate the values here
tags[key] = evaluate_template(value, info, [], args, account_info)
tags.update({
"Name": stack_name,
"StackName": data['Mappings']['Senza']['Info']['StackName'],
"StackVersion": data['Mappings']['Senza']['Info']['StackVersion']
})
tags_list = []
tags_mapping_list = []
for k, v in tags.items():
tags_list.append({'Key': k, 'Value': v})
tags_mapping_list.append({k: v})
data['Mappings']['Senza']['Info']['Tags'] = tags_mapping_list
if "OperatorTopicId" in data['Mappings']['Senza']['Info']:
topic = data['Mappings']['Senza']['Info']["OperatorTopicId"]
topic_arn = resolve_topic_arn(region, topic)
if not topic_arn:
fatal_error('Error: SNS topic "{}" does not exist'.format(topic))
topics = [topic_arn]
else:
topics = []
capabilities = get_required_capabilities(data)
cfjson = json.dumps(data, sort_keys=True, indent=4)
return {'StackName': stack_name, 'TemplateBody': cfjson, 'Parameters': parameters, 'Tags': tags_list,
'NotificationARNs': topics, 'Capabilities': capabilities}
def main(pipfile_locks: List[str]):
"""
Check whether specified Pipfile.lock file(s) are up to date with their Pipfile(s).
If no Pipfile.lock paths are provided, the current directory is assumed.
"""
if not pipfile_locks:
pipfile_locks = [Path(os.getcwd()) / "Pipfile.lock"]
for pipfile_lock in pipfile_locks:
pipfile_dir: Path = Path(pipfile_lock).parent
if not check_dir(pipfile_dir):
cc.fatal_error(f"{pipfile_lock} is out of date. Consider running 'pipenv lock' or 'pipenv install'")
def detect_etcd_discovery_domain_for_region(dbaas_zone, user_region):
""" Query DNS zone for the etcd record corresponding to a given region. """
user_region = user_region.split('-')[1] # leave only 'west' out of 'eu-west-1'
records = get_records_for_hosted_zone(dbaas_zone)
if not records:
fatal_error("Unable to list records for {0}: make sure you are logged into the DBaaS account".
format(dbaas_zone))
for r in records['ResourceRecordSets']:
if r['Type'] == 'SRV' and r['Name'] == '_etcd._tcp.{region}.{zone}'.format(region=user_region,
zone=dbaas_zone):
return "{region}.{zone}".format(region=user_region,
zone=dbaas_zone[:-1])
return None
def sli_get(obj, product_name, name):
"""Get SLI for a product by name"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
slis = client.sli_list(product[0], name=name)
if not slis:
fatal_error('SLI {} does not exist'.format(name))
print(json.dumps(slis[0], indent=4))
def test_nginx(run_id, url, token):
manifest = '''
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-{run_id}
spec:
replicas: 1
template:
metadata:
labels:
app: nginx-{run_id}
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
'''.format(run_id=run_id)
create_deployment(manifest, url, token)
manifest = '''
kind: Service
apiVersion: v1
metadata:
name: nginx-{run_id}
spec:
selector:
app: nginx-{run_id}
type: LoadBalancer
ports:
- port: 80
targetPort: 80
protocol: TCP
name: http
'''.format(run_id=run_id)
create_service(manifest, url, token)
available = wait_for_deployment('nginx-{}'.format(run_id), url, token)
if not available:
fatal_error('Deployment failed')
parts = urllib.parse.urlsplit(url)
domain = parts.netloc.split('.', 1)[-1]
available = wait_for_resource('http://nginx-{}-e2e.{}/'.format(
run_id, domain),
'Welcome to nginx!',
timeout=300)
if not available:
fatal_error('ELB service registration failed')
def main(url, token):
run_id = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(8))
info('Starting test run {}..'.format(run_id))
all_containers_ready = False
with Action('Waiting for all containers to be ready..') as act:
for i in range(60):
containers = get_containers(url, token)
ready = True
for name in EXPECTED_CONTAINERS:
if not containers.get(name, {}).get('ready'):
info('{} is not ready yet (restarts: {})'.format(
name,
containers.get(name, {}).get('restart_count')))
ready = False
if ready:
all_containers_ready = True
break
time.sleep(5)
act.progress()
if not all_containers_ready:
fatal_error('Not all containers are ready')
manifest = '''
apiVersion: v1
kind: Namespace
metadata:
name: e2e
'''
try:
create_resource(manifest, url + '/api/v1/namespaces', token)
except requests.exceptions.HTTPError as e:
# it's ok if the namespace is already there (409 Conflict)
if e.response.status_code != 409:
raise
for entry in os.listdir('tests'):
if entry.startswith('test_'):
module_name = entry.split('.')[0]
module = importlib.import_module('tests.{}'.format(module_name))
func = getattr(module, module_name)
info('Running {}..'.format(module_name))
func(run_id, url, token)
def detect_zmon_security_group(region):
ec2 = boto3.client('ec2', region)
filters = [{'Name': 'tag-key', 'Values': ['StackName']}, {'Name': 'tag-value', 'Values': ['zmon-appliance']}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get('Reservations', []):
for instance in reservation.get('Instances', []):
zmon_sgs += [sg['GroupId'] for sg in instance.get('SecurityGroups', []) if 'zmon' in sg['GroupName']]
if len(zmon_sgs) == 0:
fatal_error('Could not find zmon security group')
if len(zmon_sgs) > 1:
fatal_error("More than one security group found for zmon")
return zmon_sgs[0]
def test_echo():
action('Action..')
ok()
action('Action..')
error(' some error')
action('Action..')
with pytest.raises(SystemExit):
fatal_error(' some fatal error') # noqa
action('Action..')
warning(' some warning')
info('Some info')
def test_echo():
action('Action..')
ok()
action('Action..')
error(' some error')
action('Action..')
with pytest.raises(SystemExit):
fatal_error(' some fatal error') # noqa
action('Action..')
warning(' some warning')
info('Some info')
def sli_values(obj, product_name, name, count):
"""List SLI values"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
slis = client.sli_list(product, name)
if not slis:
fatal_error('SLI {} does not exist'.format(name))
res = client.sli_values(slis[0], page_size=count)
print(json.dumps(res, indent=4))
def agent_error(e: requests.HTTPError, fatal=True):
"""
Prints an agent error and exits
"""
data = e.response.json()
details = data['detail'] # type: str
if details:
lines = ('[AGENT] {}'.format(line) for line in details.splitlines())
msg = '\n' + '\n'.join(lines)
else:
msg = "[AGENT] {status} {title}".format_map(data)
if fatal:
fatal_error(msg)
else:
error(msg)
def agent_error(e: requests.HTTPError, fatal=True):
"""
Prints an agent error and exits
"""
try:
data = e.response.json()
details = data['detail'] # type: str
except JSONDecodeError:
details = e.response.text or str(e.response)
lines = ('[AGENT] {}'.format(line) for line in details.splitlines())
msg = '\n' + '\n'.join(lines)
if fatal:
fatal_error(msg)
else:
error(msg)
def detect_eu_team_odd_instances(team_zone_name):
"""
Detect the odd instances by name. Same reliance on a convention as with
the detect_eu_team_nat_gateways.
"""
resolver = dns.resolver.Resolver()
odd_hosts = []
for region in ('eu-west-1', 'eu-central-1'):
try:
answer = resolver.query('odd-{region}.{zone}'.format(region=region, zone=team_zone_name))
odd_hosts.extend(str(rdata) for rdata in answer)
except dns.resolver.NXDOMAIN:
continue
if not odd_hosts:
fatal_error("Unable to detect odd hosts: make sure {0} account is set up correctly".format(team_zone_name))
return odd_hosts
def target_delete(obj, product_name, target_uri):
"""Delete Target for certain product"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
target = client.target_get(target_uri)
if product['name'] != target['product_name']:
fatal_error('Cannot delete Target {} as it does not belong to product {}'.format(target_uri, product_name))
with Action('Deleting Target: {}'.format(target_uri), nl=True):
client.target_delete(target)
def report_create(obj, product_name, output_dir):
"""Create report for a product"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
try:
subprocess.check_output(['which', 'gnuplot'])
except subprocess.CalledProcessError:
fatal_error('Missing system dependency. Please install *gnuplot* system package!')
with Action('Creating report for product: {}'.format(product_name), nl=True):
generate_weekly_report(client, product, output_dir)
def detect_eu_team_nat_gateways(team_zone_name):
"""
Detect NAT gateways. Since the complete zone is not hosted by DBaaS and
not accessible, try to figure out individual NAT endpoints by name.
"""
resolver = dns.resolver.Resolver()
nat_gateways = []
for region in ('eu-west-1', 'eu-central-1'):
for az in ('a', 'b', 'c'):
try:
answer = resolver.query('nat-{region}{az}.{zone}'.
format(region=region, az=az, zone=team_zone_name), 'A')
nat_gateways.extend(str(rdata) for rdata in answer)
except dns.resolver.NXDOMAIN:
continue
if not nat_gateways:
fatal_error("Unable to detect nat gateways: make sure {0} account is set up correctly".format(team_zone_name))
return nat_gateways
def sli_create(obj, product_name, sli_file):
"""Create SLI for a product"""
client = get_client(obj)
product = client.product_list(name=product_name)
if not product:
fatal_error('Product {} does not exist'.format(product_name))
product = product[0]
with Action('Creating SLI for product: {}'.format(product_name), nl=True) as act:
sli = json.load(sli_file)
validate_sli(obj, sli, act)
if not act.errors:
res = client.sli_create(product, sli['name'], sli['unit'], sli['source'])
print(json.dumps(res, indent=4))
def login(config, url):
'''Login to Pier One Docker registry (generates docker configuration in ~/.docker/config.json)'''
url_option_was_set = url
url = set_pierone_url(config, url)
if not url_option_was_set:
stups_cli.config.store_config(config, 'pierone')
# Check if the credential helper is available
if shutil.which("docker-credential-pierone") is None:
fatal_error(
"docker-credential-pierone executable is not available. "
"If you've installed `pierone` to a virtual environment, make sure to add it to to the PATH."
)
docker_login_with_credhelper(url)
ok("Authentication configured for {}, you don't need to run pierone login anymore!"
.format(url))
def product_group_update(obj, name, new_name, department):
"""Update product group"""
client = get_client(obj)
pgs = client.product_group_list(name)
if not pgs:
fatal_error('Product group {} does not exist'.format(name))
with Action('Updating product_group: {}'.format(name), nl=True):
pg = pgs[0]
if new_name:
pg['name'] = new_name
if department:
pg['department'] = department
pg = client.product_group_update(pg)
print(json.dumps(pg, indent=4))
def list_stacks(stack_ref: str, all: bool, watch: int, output: str):
"""List Lizzy stacks"""
config = Configuration()
access_token = fetch_token(config.token_url, config.scopes, config.credentials_dir)
lizzy = Lizzy(config.lizzy_url, access_token)
repeat = True
while repeat:
try:
all_stacks = lizzy.get_stacks()
except requests.RequestException as e:
fatal_error('Failed to get stacks: {}'.format(e))
if all:
stacks = all_stacks
else:
stacks = [stack for stack in all_stacks if stack['status'] not in ['LIZZY:REMOVED']]
if stack_ref:
stacks = [stack for stack in stacks if stack['stack_name'] in stack_ref]
rows = []
for stack in stacks:
creation_time = dateutil.parser.parse(stack['creation_time'])
rows.append({'stack_name': stack['stack_name'],
'version': stack['stack_version'],
'image_version': stack['image_version'],
'status': stack['status'],
'creation_time': creation_time.timestamp()})
rows.sort(key=lambda x: (x['stack_name'], x['version']))
with OutputFormat(output):
print_table('stack_name version image_version status creation_time'.split(),
rows, styles=STYLES, titles=TITLES)
if watch: # pragma: no cover
time.sleep(watch)
click.clear()
else:
repeat = False
def setup_lizzy_client(explicit_agent_url=None):
config = Configuration()
try:
token_url = config.token_url
except AttributeError:
fatal_error('Environment variable OAUTH2_ACCESS_TOKEN_URL is not set.')
scopes = config.scopes
credentials_dir = config.credentials_dir
access_token = fetch_token(token_url, scopes, credentials_dir)
try:
lizzy_url = explicit_agent_url or config.lizzy_url
except AttributeError:
fatal_error('Environment variable LIZZY_URL is not set.')
return Lizzy(lizzy_url, access_token)
def delete(stack_ref, region, dry_run, force):
'''Delete a single Cloud Formation stack'''
stack_refs = get_stack_refs(stack_ref)
region = get_region(region)
check_credentials(region)
cf = boto3.client('cloudformation', region)
if not stack_refs:
raise click.UsageError('Please specify at least one stack')
stacks = list(get_stacks(stack_refs, region))
if not all_with_version(stack_refs) and len(stacks) > 1 and not dry_run and not force:
fatal_error('Error: {} matching stacks found. '.format(len(stacks)) +
'Please use the "--force" flag if you really want to delete multiple stacks.')
for stack in stacks:
with Action('Deleting Cloud Formation stack {}..'.format(stack.StackName)):
if not dry_run:
cf.delete_stack(StackName=stack.StackName)
def get_listeners(subdomain, main_zone, configuration,
account_info: AccountArguments):
ssl_cert = configuration.get('SSLCertificateId')
if ACMCertificate.arn_is_acm_certificate(ssl_cert):
# check if certificate really exists
try:
ACMCertificate.get_by_arn(account_info.Region, ssl_cert)
except ClientError as e:
error_msg = e.response['Error']['Message']
fatal_error(error_msg)
elif IAMServerCertificate.arn_is_server_certificate(ssl_cert):
# TODO check if certificate exists
pass
elif ssl_cert is not None:
certificate = IAMServerCertificate.get_by_name(ssl_cert)
ssl_cert = certificate.arn
elif main_zone is not None:
if main_zone:
iam_pattern = main_zone.lower().rstrip('.').replace('.', '-')
name = '{sub}.{zone}'.format(sub=subdomain,
zone=main_zone.rstrip('.'))
acm = ACM(account_info.Region)
acm_certificates = sorted(acm.get_certificates(domain_name=name),
reverse=True)
else:
iam_pattern = ''
acm_certificates = []
iam_certificates = sorted(IAM.get_certificates(name=iam_pattern))
if not iam_certificates:
# if there are no iam certificates matching the pattern
# try to use any certificate
iam_certificates = sorted(IAM.get_certificates(), reverse=True)
# the priority is acm_certificate first and iam_certificate second
certificates = (acm_certificates +
iam_certificates) # type: List[Union[ACMCertificate, IAMServerCertificate]]
try:
certificate = certificates[0]
ssl_cert = certificate.arn
except IndexError:
if main_zone:
fatal_error('Could not find any matching '
'SSL certificate for "{}"'.format(name))
else:
fatal_error('Could not find any SSL certificate')
return [
{
"PolicyNames": [],
"SSLCertificateId": ssl_cert,
"Protocol": "HTTPS",
"InstancePort": configuration["HTTPPort"],
"LoadBalancerPort": 443
}
]
def get_ssl_cert(subdomain, main_zone, configuration, account_info: AccountArguments):
ssl_cert = configuration.get("SSLCertificateId")
if ACMCertificate.arn_is_acm_certificate(ssl_cert):
# check if certificate really exists
try:
ACMCertificate.get_by_arn(account_info.Region, ssl_cert)
except ClientError as e:
error_msg = e.response["Error"]["Message"]
fatal_error(error_msg)
elif IAMServerCertificate.arn_is_server_certificate(ssl_cert):
# TODO check if certificate exists
pass
elif ssl_cert is not None:
certificate = IAMServerCertificate.get_by_name(account_info.Region, ssl_cert)
ssl_cert = certificate.arn
elif main_zone is not None:
if main_zone:
iam_pattern = main_zone.lower().rstrip(".").replace(".", "-")
name = "{sub}.{zone}".format(sub=subdomain, zone=main_zone.rstrip("."))
acm = ACM(account_info.Region)
acm_certificates = sorted(acm.get_certificates(domain_name=name), reverse=True)
else:
iam_pattern = ""
acm_certificates = []
iam = IAM(account_info.Region)
iam_certificates = sorted(iam.get_certificates(name=iam_pattern))
if not iam_certificates:
# if there are no iam certificates matching the pattern
# try to use any certificate
iam_certificates = sorted(iam.get_certificates(), reverse=True)
# the priority is acm_certificate first and iam_certificate second
certificates = acm_certificates + iam_certificates # type: List[Union[ACMCertificate, IAMServerCertificate]]
try:
certificate = certificates[0]
ssl_cert = certificate.arn
except IndexError:
if main_zone:
fatal_error("Could not find any matching " 'SSL certificate for "{}"'.format(name))
else:
fatal_error("Could not find any SSL certificate")
return ssl_cert
def create(definition: str, image_version: str, keep_stacks: int,
traffic: int, verbose: bool, senza_parameters: list,
app_version: Optional[str], stack_version: Optional[str],
disable_rollback: bool):
senza_parameters = senza_parameters or []
config = Configuration()
access_token = fetch_token(config.token_url, config.scopes, config.credentials_dir)
lizzy = Lizzy(config.lizzy_url, access_token)
with Action('Requesting new stack..') as action:
try:
stack_id = lizzy.new_stack(image_version, keep_stacks, traffic,
definition, stack_version, app_version,
disable_rollback, senza_parameters)
except requests.RequestException as e:
action.fatal_error('Deployment failed: {}.'.format(e))
info('Stack ID: {}'.format(stack_id))
with Action('Waiting for new stack...') as action:
if verbose:
print() # ensure that new states will not be printed on the same line as the action
last_state = None
for state in lizzy.wait_for_deployment(stack_id):
if state != last_state and verbose:
click.echo(' {}'.format(state))
else:
action.progress()
last_state = state
if last_state == 'CF:ROLLBACK_COMPLETE':
fatal_error('Stack was rollback after deployment. Check you application log for possible reasons.')
elif last_state == 'LIZZY:REMOVED':
fatal_error('Stack was removed before deployment finished.')
elif last_state != 'CF:CREATE_COMPLETE':
fatal_error('Deployment failed: {}'.format(last_state))
info('Deployment Successful')
import click
from clickclick import Action, action, fatal_error, ok, print_table, warning
from .aws import StackReference, get_stacks, get_tag
from .manaus import ClientError
from .manaus.boto_proxy import BotoClientProxy
from .manaus.cloudformation import CloudFormationStack, ResourceType
from .manaus.exceptions import ELBNotFound, StackNotFound, StackNotUpdated
from .manaus.route53 import (RecordType, Route53, Route53HostedZone,
convert_cname_records_to_alias)
from .manaus.utils import extract_client_error_code
try:
import dns.resolver
except ImportError:
fatal_error("Failed to import dns.resolver.\n"
"Run 'pip3 install -U --force-reinstall dnspython'.")
PERCENT_RESOLUTION = 2
FULL_PERCENTAGE = PERCENT_RESOLUTION * 100
DNS_RR_CACHE = {}
DNS_ZONE_CACHE = {}
def get_weights(dns_names: list, identifier: str, all_identifiers) -> ({str: int}, int, int):
"""
For the given dns_name, get the dns record weights from provided dns record set
followed by partial count and partial weight sum.
Here partial means without the element that we are operating now on.
"""
partial_count = 0
partial_sum = 0
def gather_user_variables(variables, account_info, region):
set_default_variables(variables)
missing = []
for required in ('team_name', 'team_region', 'team_gateway_zone', 'hosted_zone'):
if not variables.get(required):
missing.append(required)
if len(missing) > 0:
fatal_error("Missing values for the following variables: {0}".format(', '.join(missing)))
# redefine the region per the user input
if variables['team_region'] != region.Region:
fatal_error("Current region {0} do not match the requested region {1}\n"
"Change the currect region with --region option or set AWS_DEFAULT_REGION variable.".
format(region.Region, variables['team_region']))
variables['wal_s3_bucket'] = '{}-{}-spilo-dbaas'.format(get_account_alias(), region.Region)
for name in ('team_gateway_zone', 'hosted_zone'):
if variables[name][-1] != '.':
variables[name] += '.'
# split the ldap url into the URL and suffix (path component)
if variables['ldap_url']:
url = urlparse(variables['ldap_url'])
if url.path and url.path[0] == '/':
variables['ldap_suffix'] = url.path[1:]
# if master DNS name is specified but not the replica one - derive the replica name from the master
if variables['master_dns_name'] and not variables['replica_dns_name']:
replica_dns_components = variables['master_dns_name'].split('.')
replica_dns_components[0] += '-repl'
variables['replica_dns_name'] = '.'.join(replica_dns_components)
# make sure all DNS names belong to the hosted zone
for v in ('master_dns_name', 'replica_dns_name'):
if variables[v] and not check_dns_name(variables[v], variables['hosted_zone'][:-1]):
fatal_error("{0} should end with {1}".
format(v.replace('_', ' '), variables['hosted_zone'][:-1]))
if variables['ldap_url'] and not variables['ldap_suffix']:
fatal_error("LDAP URL is missing the suffix: shoud be in a format: "
"ldap[s]://example.com[:port]/ou=people,dc=example,dc=com")
# pick up the proper etcd address depending on the region
variables['discovery_domain'] = detect_etcd_discovery_domain_for_region(variables['hosted_zone'],
region.Region)
# get the IP addresses of the NAT gateways to acess a given ELB.
variables['nat_gateway_addresses'] = detect_eu_team_nat_gateways(variables['team_gateway_zone'])
variables['odd_instance_addresses'] = detect_eu_team_odd_instances(variables['team_gateway_zone'])
variables['spilo_security_group_ingress_rules_block'] = \
generate_spilo_master_security_group_ingress(variables['nat_gateway_addresses'] +
variables['odd_instance_addresses'])
if variables['postgresqlconf']:
variables['postgresqlconf'] = generate_postgresql_configuration(variables['postgresqlconf'])
odd_sg = get_security_group(region.Region, ODD_SG_NAME)
variables['odd_sg_id'] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
variables['zmon_sg_id'] = detect_zmon_security_group(region.Region)
if variables['volume_type'] == 'io1' and not variables['volume_iops']:
pio_max = variables['volume_size'] * 30
variables['volume_iops'] = str(pio_max)
variables['ebs_optimized'] = ebs_optimized_supported(variables['instance_type'])
# pick up the first key with a description containing spilo
kms_keys = [k for k in list_kms_keys(region.Region)
if 'alias/aws/ebs' not in k['aliases'] and 'spilo' in ((k['Description']).lower())]
if len(kms_keys) == 0:
raise fatal_error('No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
kms_key = kms_keys[0]
kms_keyid = kms_key['KeyId']
variables['kms_arn'] = kms_key['Arn']
for key in [k for k in variables if k.startswith('pgpassword_')] +\
(['scalyr_account_key'] if variables.get('scalyr_account_key') else []):
encrypted = encrypt(region=region.Region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
check_s3_bucket(variables['wal_s3_bucket'], region.Region)
return variables
def component_elastic_load_balancer(definition, configuration, args, info, force, account_info):
lb_name = configuration["Name"]
# domains pointing to the load balancer
main_zone = None
for name, domain in configuration.get('Domains', {}).items():
name = '{}{}'.format(lb_name, name)
definition["Resources"][name] = {
"Type": "AWS::Route53::RecordSet",
"Properties": {
"Type": "CNAME",
"TTL": 20,
"ResourceRecords": [
{"Fn::GetAtt": [lb_name, "DNSName"]}
],
"Name": "{0}.{1}".format(domain["Subdomain"], domain["Zone"]),
"HostedZoneName": "{0}".format(domain["Zone"])
},
}
if domain["Type"] == "weighted":
definition["Resources"][name]["Properties"]['Weight'] = 0
definition["Resources"][name]["Properties"]['SetIdentifier'] = "{0}-{1}".format(info["StackName"],
info["StackVersion"])
main_zone = domain['Zone']
ssl_cert = configuration.get('SSLCertificateId')
pattern = None
if not ssl_cert:
if main_zone:
pattern = main_zone.lower().rstrip('.').replace('.', '-')
else:
pattern = ''
elif not ssl_cert.startswith('arn:'):
pattern = ssl_cert
if pattern is not None:
ssl_cert = find_ssl_certificate_arn(args.region, pattern)
if not ssl_cert:
fatal_error('Could not find any matching SSL certificate for "{}"'.format(pattern))
health_check_protocol = "HTTP"
allowed_health_check_protocols = ("HTTP", "TCP", "UDP", "SSL")
if "HealthCheckProtocol" in configuration:
health_check_protocol = configuration["HealthCheckProtocol"]
if health_check_protocol not in allowed_health_check_protocols:
raise click.UsageError('Protocol "{}" is not supported for LoadBalancer'.format(health_check_protocol))
health_check_path = "/ui/"
if "HealthCheckPath" in configuration:
health_check_path = configuration["HealthCheckPath"]
health_check_port = configuration["HTTPPort"]
if "HealthCheckPort" in configuration:
health_check_port = configuration["HealthCheckPort"]
health_check_target = "{0}:{1}{2}".format(health_check_protocol, health_check_port, health_check_path)
if configuration.get('NameSufix'):
loadbalancer_name = get_load_balancer_name(info["StackName"], '{}-{}'.format(info["StackVersion"],
configuration['NameSufix']))
del(configuration['NameSufix'])
else:
loadbalancer_name = get_load_balancer_name(info["StackName"], info["StackVersion"])
loadbalancer_scheme = "internal"
allowed_loadbalancer_schemes = ("internet-facing", "internal")
if "Scheme" in configuration:
loadbalancer_scheme = configuration["Scheme"]
else:
configuration["Scheme"] = loadbalancer_scheme
if loadbalancer_scheme == 'internet-facing':
click.secho('You are deploying an internet-facing ELB that will be publicly accessible! ' +
'You should have OAUTH2 and HTTPS in place!', fg='red', bold=True, err=True)
if loadbalancer_scheme not in allowed_loadbalancer_schemes:
raise click.UsageError('Scheme "{}" is not supported for LoadBalancer'.format(loadbalancer_scheme))
if loadbalancer_scheme == "internal":
loadbalancer_subnet_map = "LoadBalancerInternalSubnets"
else:
loadbalancer_subnet_map = "LoadBalancerSubnets"
# load balancer
definition["Resources"][lb_name] = {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"Subnets": {"Fn::FindInMap": [loadbalancer_subnet_map, {"Ref": "AWS::Region"}, "Subnets"]},
"HealthCheck": {
"HealthyThreshold": "2",
"UnhealthyThreshold": "2",
"Interval": "10",
"Timeout": "5",
"Target": health_check_target
},
"Listeners": [
{
"PolicyNames": [],
"SSLCertificateId": ssl_cert,
"Protocol": "HTTPS",
"InstancePort": configuration["HTTPPort"],
"LoadBalancerPort": 443
}
],
"CrossZone": "true",
"LoadBalancerName": loadbalancer_name,
"SecurityGroups": resolve_security_groups(configuration["SecurityGroups"], args.region),
"Tags": [
# Tag "Name"
{
"Key": "Name",
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"Value": info["StackVersion"]
}
]
}
}
for key, val in configuration.items():
# overwrite any specified properties, but
# ignore our special Senza properties as they are not supported by CF
if key not in SENZA_PROPERTIES:
definition['Resources'][lb_name]['Properties'][key] = val
return definition
def create(definition: dict, version: str, parameter: tuple,
region: str,
disable_rollback: bool,
dry_run: bool,
force: bool,
tag: List[str],
timeout: int,
keep_stacks: Optional[int],
traffic: int,
verbose: bool,
remote: str,
parameter_file: Optional[str]
):
"""
Create a new Cloud Formation stack from the given Senza definition file
"""
lizzy = setup_lizzy_client(remote)
parameter = list(parameter) or []
if parameter_file:
parameter.extend(read_parameter_file(parameter_file))
if not force: # pragma: no cover
# supporting artifact checking would imply copying a large amount of code
# from senza, so it should be considered out of scope until senza
# and lizzy client are merged
warning("WARNING: "
"Artifact checking is still not supported by lizzy-client.")
with Action('Requesting new stack..') as action:
new_stack, output = lizzy.new_stack(keep_stacks, traffic,
definition, version,
disable_rollback, parameter,
region=region,
dry_run=dry_run,
tags=tag)
stack_id = '{stack_name}-{version}'.format_map(new_stack)
print(output)
info('Stack ID: {}'.format(stack_id))
if dry_run:
info("Post deployment steps skipped")
exit(0)
with Action('Waiting for new stack...') as action:
if verbose:
print() # ensure that new states will not be printed on the same line as the action
last_state = None
for state in lizzy.wait_for_deployment(stack_id, region=region):
if state != last_state and verbose:
click.echo(' {}'.format(state))
else:
action.progress()
last_state = state
# TODO be prepared to handle all final AWS CF states
if last_state == 'ROLLBACK_COMPLETE':
fatal_error(
'Stack was rollback after deployment. Check your application log for possible reasons.')
elif last_state != 'CREATE_COMPLETE':
fatal_error('Deployment failed: {}'.format(last_state))
info('Deployment Successful')
if traffic is not None:
with Action('Requesting traffic change..'):
try:
lizzy.traffic(stack_id, traffic, region=region)
except requests.ConnectionError as e:
connection_error(e, fatal=False)
except requests.HTTPError as e:
agent_error(e, fatal=False)
# TODO unit test this
if keep_stacks is not None:
versions_to_keep = keep_stacks + 1
stacks_to_remove_counter = 1
end_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=timeout)
while stacks_to_remove_counter > 0 and datetime.datetime.utcnow() <= end_time:
try:
all_stacks = lizzy.get_stacks([new_stack['stack_name']],
region=region)
except requests.ConnectionError as e:
connection_error(e, fatal=False)
error("Failed to fetch old stacks. "
"Old stacks WILL NOT BE DELETED")
exit(1)
except requests.HTTPError as e:
agent_error(e, fatal=False)
error("Failed to fetch old stacks. "
"Old stacks WILL NOT BE DELETED")
exit(1)
else:
sorted_stacks = sorted(all_stacks,
key=lambda stack: stack['creation_time'])
stacks_to_remove = sorted_stacks[:-versions_to_keep]
stacks_to_remove_counter = len(stacks_to_remove)
with Action('Deleting old stacks..'):
print()
for old_stack in stacks_to_remove:
old_stack_id = '{stack_name}-{version}'.format_map(
old_stack)
if old_stack['status'] in COMPLETE_STATES:
click.echo(' {}'.format(old_stack_id))
try:
lizzy.delete(old_stack_id, region=region)
stacks_to_remove_counter -= 1
except requests.ConnectionError as e:
connection_error(e, fatal=False)
except requests.HTTPError as e:
agent_error(e, fatal=False)
else:
click.echo(' > {} current status is {} trying '
'again later'.format(old_stack_id,
old_stack['status']))
if stacks_to_remove_counter > 0:
time.sleep(5)
if datetime.datetime.utcnow() > end_time:
click.echo('Timeout waiting for related stacks to be ready.')
def component_elastic_load_balancer(definition, configuration, args, info, force, account_info):
lb_name = configuration["Name"]
# domains pointing to the load balancer
subdomain = ''
main_zone = None
for name, domain in configuration.get('Domains', {}).items():
name = '{}{}'.format(lb_name, name)
definition["Resources"][name] = {
"Type": "AWS::Route53::RecordSet",
"Properties": {
"Type": "CNAME",
"TTL": 20,
"ResourceRecords": [
{"Fn::GetAtt": [lb_name, "DNSName"]}
],
"Name": "{0}.{1}".format(domain["Subdomain"], domain["Zone"]),
"HostedZoneName": "{0}".format(domain["Zone"])
},
}
if domain["Type"] == "weighted":
definition["Resources"][name]["Properties"]['Weight'] = 0
definition["Resources"][name]["Properties"]['SetIdentifier'] = "{0}-{1}".format(info["StackName"],
info["StackVersion"])
subdomain = domain['Subdomain']
main_zone = domain['Zone'] # type: str
ssl_cert = configuration.get('SSLCertificateId')
if ACMCertificate.arn_is_acm_certificate(ssl_cert):
# check if certificate really exists
try:
ACMCertificate.get_by_arn(ssl_cert)
except ClientError as e:
error_msg = e.response['Error']['Message']
fatal_error(error_msg)
elif IAMServerCertificate.arn_is_server_certificate(ssl_cert):
# TODO check if certificate exists
pass
elif ssl_cert is not None:
certificate = IAMServerCertificate.get_by_name(ssl_cert)
ssl_cert = certificate.arn
elif main_zone is not None:
if main_zone:
iam_pattern = main_zone.lower().rstrip('.').replace('.', '-')
name = '{sub}.{zone}'.format(sub=subdomain,
zone=main_zone.rstrip('.'))
acm_certificates = sorted(ACM.get_certificates(domain_name=name),
reverse=True)
else:
iam_pattern = ''
acm_certificates = []
iam_certificates = sorted(IAM.get_certificates(name=iam_pattern))
if not iam_certificates:
# if there are no iam certificates matching the pattern
# try to use any certificate
iam_certificates = sorted(IAM.get_certificates(), reverse=True)
# the priority is acm_certificate first and iam_certificate second
certificates = (acm_certificates +
iam_certificates) # type: List[Union[ACMCertificate, IAMServerCertificate]]
try:
certificate = certificates[0]
ssl_cert = certificate.arn
except IndexError:
if main_zone:
fatal_error('Could not find any matching '
'SSL certificate for "{}"'.format(name))
else:
fatal_error('Could not find any SSL certificate')
health_check_protocol = "HTTP"
allowed_health_check_protocols = ("HTTP", "TCP", "UDP", "SSL")
if "HealthCheckProtocol" in configuration:
health_check_protocol = configuration["HealthCheckProtocol"]
if health_check_protocol not in allowed_health_check_protocols:
raise click.UsageError('Protocol "{}" is not supported for LoadBalancer'.format(health_check_protocol))
health_check_path = "/ui/"
if "HealthCheckPath" in configuration:
health_check_path = configuration["HealthCheckPath"]
health_check_port = configuration["HTTPPort"]
if "HealthCheckPort" in configuration:
health_check_port = configuration["HealthCheckPort"]
health_check_target = "{0}:{1}{2}".format(health_check_protocol,
health_check_port,
health_check_path)
if configuration.get('LoadBalancerName'):
loadbalancer_name = info["LoadBalancerName"]
elif configuration.get('NameSuffix'):
version = '{}-{}'.format(info["StackVersion"],
configuration['NameSuffix'])
loadbalancer_name = get_load_balancer_name(info["StackName"], version)
del(configuration['NameSuffix'])
else:
loadbalancer_name = get_load_balancer_name(info["StackName"],
info["StackVersion"])
loadbalancer_scheme = "internal"
allowed_loadbalancer_schemes = ("internet-facing", "internal")
if "Scheme" in configuration:
loadbalancer_scheme = configuration["Scheme"]
else:
configuration["Scheme"] = loadbalancer_scheme
if loadbalancer_scheme == 'internet-facing':
click.secho('You are deploying an internet-facing ELB that will be '
'publicly accessible! You should have OAUTH2 and HTTPS '
'in place!', bold=True, err=True)
if loadbalancer_scheme not in allowed_loadbalancer_schemes:
raise click.UsageError('Scheme "{}" is not supported for LoadBalancer'.format(loadbalancer_scheme))
if loadbalancer_scheme == "internal":
loadbalancer_subnet_map = "LoadBalancerInternalSubnets"
else:
loadbalancer_subnet_map = "LoadBalancerSubnets"
# load balancer
definition["Resources"][lb_name] = {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"Subnets": {"Fn::FindInMap": [loadbalancer_subnet_map, {"Ref": "AWS::Region"}, "Subnets"]},
"HealthCheck": {
"HealthyThreshold": "2",
"UnhealthyThreshold": "2",
"Interval": "10",
"Timeout": "5",
"Target": health_check_target
},
"Listeners": [
{
"PolicyNames": [],
"SSLCertificateId": ssl_cert,
"Protocol": "HTTPS",
"InstancePort": configuration["HTTPPort"],
"LoadBalancerPort": 443
}
],
"ConnectionDrainingPolicy": {
"Enabled": True,
"Timeout": 60
},
"CrossZone": "true",
"LoadBalancerName": loadbalancer_name,
"SecurityGroups": resolve_security_groups(configuration["SecurityGroups"], args.region),
"Tags": [
# Tag "Name"
{
"Key": "Name",
"Value": "{0}-{1}".format(info["StackName"], info["StackVersion"])
},
# Tag "StackName"
{
"Key": "StackName",
"Value": info["StackName"],
},
# Tag "StackVersion"
{
"Key": "StackVersion",
"Value": info["StackVersion"]
}
]
}
}
for key, val in configuration.items():
# overwrite any specified properties, but
# ignore our special Senza properties as they are not supported by CF
if key not in SENZA_PROPERTIES:
definition['Resources'][lb_name]['Properties'][key] = val
return definition
def run(spec_file,
base_module_path,
port,
host,
wsgi_server,
server,
stub,
mock,
hide_spec,
hide_console_ui,
console_ui_url,
console_ui_from,
auth_all_paths,
validate_responses,
strict_validation,
debug,
verbose,
base_path,
app_framework):
"""
Runs a server compliant with a OpenAPI/Swagger 2.0 Specification file.
Arguments:
- SPEC_FILE: specification file that describes the server endpoints.
- BASE_MODULE_PATH (optional): filesystem path where the API endpoints handlers are going to be imported from.
"""
if wsgi_server and server:
raise click.BadParameter(
"these options are mutually exclusive",
param_hint="'wsgi-server' and 'server'"
)
elif wsgi_server:
server = wsgi_server
if server is None:
server = DEFAULT_SERVERS[app_framework]
if app_framework not in AVAILABLE_SERVERS[server]:
message = "Invalid server '{}' for app-framework '{}'".format(
server, app_framework
)
raise click.UsageError(message)
if app_framework == AIOHTTP_APP:
try:
import aiohttp # NOQA
except Exception:
fatal_error('aiohttp library is not installed')
logging_level = logging.WARN
if verbose > 0:
logging_level = logging.INFO
if debug or verbose > 1:
logging_level = logging.DEBUG
debug = True
logging.basicConfig(level=logging_level)
spec_file_full_path = path.abspath(spec_file)
py_module_path = base_module_path or path.dirname(spec_file_full_path)
sys.path.insert(1, path.abspath(py_module_path))
logger.debug('Added {} to system path.'.format(py_module_path))
resolver_error = None
if stub:
resolver_error = 501
api_extra_args = {}
if mock:
resolver = MockResolver(mock_all=mock == 'all')
api_extra_args['resolver'] = resolver
app_cls = connexion.utils.get_function_from_name(
AVAILABLE_APPS[app_framework]
)
options = {
"serve_spec": not hide_spec,
"swagger_path": console_ui_from or None,
"swagger_ui": not hide_console_ui,
"swagger_url": console_ui_url or None
}
app = app_cls(__name__,
debug=debug,
auth_all_paths=auth_all_paths,
options=options)
app.add_api(spec_file_full_path,
base_path=base_path,
resolver_error=resolver_error,
validate_responses=validate_responses,
strict_validation=strict_validation,
**api_extra_args)
app.run(port=port,
host=host,
server=server,
debug=debug)
def die_fatal_error(message):
"""Sent error message to stderr, in red, and exit"""
fatal_error(message, err=True)
