def choicetest():
print('>>{}<<'.format(choice('Please choose', ['a', 'b', 'c'],
default='c')))
print('>>{}<<'.format(
choice('Please choose', [('a', 'Label A'), ('b', 'Label B'),
('c', 'Label C')],
default='b')))
print('>>{}<<'.format(
choice('Please choose', [('a', 'Label A'), ('b', 'Label B'),
('c', 'Label C')],
default='x')))
def test_choice(monkeypatch):
def get_number():
yield 50
while True:
yield 1
generator = get_number()
def returnnumber(*args, **vargs):
return next(generator)
monkeypatch.setattr('click.prompt', returnnumber)
assert 'a' == choice('Please choose', ['a', 'b'])
assert 'a' == choice('Please choose', [('a', 'Label A')])
def test_choice(monkeypatch):
def get_number():
yield 50
while True:
yield 1
generator = get_number()
def returnnumber(*args, **vargs):
return next(generator)
monkeypatch.setattr('click.prompt', returnnumber)
assert 'a' == choice('Please choose', ['a', 'b'])
assert 'a' == choice('Please choose', [('a', 'Label A')])
def create(obj, profile_name, url, user):
'''Create a new profile'''
if not url.startswith('http'):
url = 'https://{}'.format(url)
saml_xml, roles = saml_login(user, url)
if not roles:
error('No roles found')
exit(1)
if len(roles) == 1:
role = roles[0]
if role[2] is None:
role = (role[0], role[1], profile_name)
else:
role = choice('Please select one role', [(r, get_role_label(r)) for r in sorted(roles)])
data = obj['config']
if not data:
data = {}
data[profile_name] = {
'saml_identity_provider_url': url,
'saml_role': role,
'saml_user': user
}
path = obj['config-file']
with Action('Storing new profile in {}..'.format(path)):
os.makedirs(obj['config-dir'], exist_ok=True)
with open(path, 'w') as fd:
yaml.safe_dump(data, fd)
def create(obj, profile_name, url, user):
'''Create a new profile'''
if not url.startswith('http'):
url = 'https://{}'.format(url)
saml_xml, roles = saml_login(user, url)
if not roles:
error('No roles found')
exit(1)
if len(roles) == 1:
role = roles[0]
if role[2] is None:
role = (role[0], role[1], profile_name)
else:
role = choice('Please select one role', [(r, get_role_label(r)) for r in sorted(roles)])
data = obj['config']
if not data:
data = {}
data[profile_name] = {
'saml_identity_provider_url': url,
'saml_role': role,
'saml_user': user
}
path = obj['config-file']
with Action('Storing new profile in {}..'.format(path)):
os.makedirs(obj['config-dir'], exist_ok=True)
with open(path, 'w') as fd:
yaml.safe_dump(data, fd)
def create(obj, profile_name, url, user):
"""Create a new profile"""
if not url.startswith("http"):
url = "https://{}".format(url)
saml_xml, roles = saml_login(user, url)
if not roles:
error("No roles found")
exit(1)
if len(roles) == 1:
role = roles[0]
if role[2] is None:
role = (role[0], role[1], profile_name)
else:
role = choice("Please select one role", [(r, get_role_label(r)) for r in sorted(roles)])
data = obj["config"]
if not data:
data = {}
data[profile_name] = {"saml_identity_provider_url": url, "saml_role": role, "saml_user": user}
path = obj["config-file"]
with Action("Storing new profile in {}..".format(path)):
os.makedirs(obj["config-dir"], exist_ok=True)
with open(path, "w") as fd:
yaml.safe_dump(data, fd)
def work_done(percentage):
'''Work done in ?? %'''
state = choice('Please select the state of your work',
['Done', 'In Progress', 'unknown', 'lost'],
default='lost')
print('Your work is {}% {}'.format(percentage, state))
def choice(variables: dict, var_name, *args, **kwargs):
if var_name not in variables:
if callable(kwargs.get('default')):
# evaluate callable
kwargs['default'] = kwargs['default']()
variables[var_name] = clickclick.choice(*args, **kwargs)
def init(definition_file, region, template, user_variable):
'''Initialize a new Senza definition'''
region = get_region(region)
check_credentials(region)
account_info = AccountArguments(region=region)
templates = []
for mod in os.listdir(os.path.join(os.path.dirname(__file__),
'templates')):
if not mod.startswith('_'):
templates.append(mod.split('.')[0])
while template not in templates:
template = choice('Please select the project template',
[(t, get_template_description(t))
for t in sorted(templates)],
default='webapp')
module = importlib.import_module('senza.templates.{}'.format(template))
variables = {}
for key_val in user_variable:
key, val = key_val
variables[key] = val
variables = module.gather_user_variables(variables, region, account_info)
with Action('Generating Senza definition file {}..'.format(
definition_file.name)):
definition = module.generate_definition(variables)
definition_file.write(definition)
def choice(variables: dict, var_name, *args, **kwargs):
if var_name not in variables:
if callable(kwargs.get("default")):
# evaluate callable
kwargs["default"] = kwargs["default"]()
variables[var_name] = clickclick.choice(*args, **kwargs)
elif "type" in kwargs:
# ensure the variable as the right type
type = kwargs["type"]
variables[var_name] = type(variables[var_name])
def choice(variables: dict, var_name, *args, **kwargs):
if var_name not in variables:
if callable(kwargs.get('default')):
# evaluate callable
kwargs['default'] = kwargs['default']()
variables[var_name] = clickclick.choice(*args, **kwargs)
elif 'type' in kwargs:
# ensure the variable as the right type
type = kwargs['type']
variables[var_name] = type(variables[var_name])
def main():
mai_accounts = mai.get_accounts()
account = choice('Select account to login', mai_accounts) # type: str
with Action("Login to pierone..") as login_action:
if not pierone.login():
login_action.fatal_error('Failed')
with Action("Login to AWS..") as login_action:
if not mai.login(account):
login_action.fatal_error('Failed')
info('Logged in to {}'.format(account))
def Domain(self):
attr = getattr(self, '__Domain', None)
if attr is None:
conn = boto3.client('route53')
domainlist = conn.list_hosted_zones()['HostedZones']
if len(domainlist) == 0:
raise AttributeError('No Domain configured')
elif len(domainlist) > 1:
domain = choice('Please select the domain',
sorted(domain['Name'] for domain in domainlist))
else:
domain = domainlist[0]['Name']
domain = conn.list_hosted_zones()['HostedZones'][0]['Name']
setattr(self, '__Domain', domain)
return domain
return attr
def __setDomain(self, domain_name=None) -> str:
"""
Sets domain for account. If there's only one hosted zone matching the
domain_name it will be used otherwise the user will be presented with
a choice.
"""
domain_list = list(Route53.get_hosted_zones(domain_name))
if len(domain_list) == 0:
raise AttributeError('No Domain configured')
elif len(domain_list) > 1:
domain = choice('Please select the domain',
sorted(domain.domain_name
for domain in domain_list))
else:
domain = domain_list[0].domain_name
self.__Domain = domain
return domain
def Domain(self):
attr = getattr(self, '__Domain', None)
if attr is None:
conn = boto3.client('route53')
domainlist = conn.list_hosted_zones()['HostedZones']
if len(domainlist) == 0:
raise AttributeError('No Domain configured')
elif len(domainlist) > 1:
domain = choice(
'Please select the domain',
sorted(domain['Name'] for domain in domainlist))
else:
domain = domainlist[0]['Name']
domain = conn.list_hosted_zones()['HostedZones'][0]['Name']
setattr(self, '__Domain', domain)
return domain
return attr
def gather_user_variables(variables, region, account_info):
# maximal 32 characters because of the loadbalancer-name
prompt(variables, 'application_id', 'Application ID', default='hello-world',
value_proc=check_value(60, '^[a-zA-Z][-a-zA-Z0-9]*$'))
prompt(variables, 'docker_image', 'Docker image without tag/version (e.g. "pierone.example.org/myteam/myapp")',
default='stups/hello-world')
prompt(variables, 'http_port', 'HTTP port', default=8080, type=int)
prompt(variables, 'http_health_check_path', 'HTTP health check path', default='/')
prompt(variables, 'instance_type', 'EC2 instance type', default='t2.micro')
if 'pierone' in variables['docker_image'] or confirm('Did you need OAuth-Credentials from Mint?'):
prompt(variables, 'mint_bucket', 'Mint S3 bucket name', default=lambda: get_mint_bucket_name(region))
else:
variables['mint_bucket'] = None
variables['loadbalancer_scheme'] = choice(prompt='Please select the load balancer scheme',
options=[('internal',
'internal: only accessible from the own VPC'),
('internet-facing',
'internet-facing: accessible from the public internet')],
default='internal')
http_port = variables['http_port']
sg_name = 'app-{}'.format(variables['application_id'])
rules_missing = check_security_group(sg_name, [('tcp', 22), ('tcp', http_port)], region, allow_from_self=True)
if ('tcp', 22) in rules_missing:
warning('Security group {} does not allow SSH access, you will not be able to ssh into your servers'.format(
sg_name))
if ('tcp', http_port) in rules_missing:
error('Security group {} does not allow inbound TCP traffic on the specified HTTP port ({})'.format(
sg_name, http_port
))
rules_missing = check_security_group(sg_name + '-lb', [('tcp', 443)], region)
if rules_missing:
error('Load balancer security group {} does not allow inbound HTTPS traffic'.format(sg_name))
check_iam_role(variables['application_id'], variables['mint_bucket'], region)
return variables
def init(definition_file, region, template, user_variable):
'''Initialize a new Senza definition'''
region = get_region(region)
check_credentials(region)
templates = []
for mod in os.listdir(os.path.join(os.path.dirname(__file__), 'templates')):
if not mod.startswith('_'):
templates.append(mod.split('.')[0])
while template not in templates:
template = choice('Please select the project template',
[(t, get_template_description(t)) for t in sorted(templates)])
module = importlib.import_module('senza.templates.{}'.format(template))
variables = {}
for key_val in user_variable:
key, val = key_val
variables[key] = val
variables = module.gather_user_variables(variables, region)
with Action('Generating Senza definition file {}..'.format(definition_file.name)):
definition = module.generate_definition(variables)
definition_file.write(definition)
def VpcID(self) -> str:
"""
Returns the VPC ID to use. If a there's a default VPC it returns that
one, otherwise it will provide the user a choice if running in an
interactive terminal or raise an exception otherwise.
"""
if self.__VpcID is None:
ec2 = EC2(self.Region)
try:
vpc = ec2.get_default_vpc()
except VPCError as error:
if sys.stdin.isatty() and error.number_of_vpcs:
# if running in interactive terminal and there are VPCs
# to choose from
vpcs = ec2.get_all_vpcs()
options = [(vpc.vpc_id, str(vpc)) for vpc in vpcs]
print("Can't find a default VPC")
vpc = choice("Select VPC to use",
options=options)
else: # if not running in interactive terminal (e.g Jenkins)
raise
self.__VpcID = vpc.vpc_id
return self.__VpcID
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm('Do you want to set the docker image now? [No]'):
prompt(variables, "docker_image", "Docker Image Version", default=get_latest_spilo_image())
prompt(variables, 'wal_s3_bucket', 'Postgres WAL S3 bucket to use',
default='{}-{}-spilo-app'.format(get_account_alias(), region))
prompt(variables, 'instance_type', 'EC2 instance type', default='t2.micro')
variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
if (variables['hosted_zone'][-1:] != '.'):
variables['hosted_zone'] += '.'
prompt(variables, 'discovery_domain', 'ETCD Discovery Domain',
default='postgres.' + variables['hosted_zone'][:-1])
if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1', 'i2', 'm3', 'r3'):
variables['use_ebs'] = click.confirm('Do you want database data directory on external (EBS) storage? [Yes]',
default=defaults['use_ebs'])
else:
variables['use_ebs'] = True
if variables['use_ebs']:
prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
pio_max = variables['volume_size'] * 30
prompt(variables, "volume_iops", 'Provisioned I/O operations per second (100 - {0})'.
format(pio_max), default=str(pio_max))
prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
if ebs_optimized_supported(variables['instance_type']):
variables['ebs_optimized'] = True
prompt(variables, "fstype", "Filesystem for the data partition", default=defaults['fstype'])
prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)",
default=defaults['fsoptions'])
prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
prompt(variables, 'pgpassword_superuser', "Password for PostgreSQL superuser [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_standby', "Password for PostgreSQL user standby [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_admin', "Password for PostgreSQL user admin", show_default=True,
default=defaults['pgpassword_admin'], hide_input=True, confirmation_prompt=True)
if click.confirm('Do you wish to encrypt these passwords using KMS?', default=False):
kms_keys = [k for k in list_kms_keys(region) if 'alias/aws/ebs' not in k['aliases']]
if len(kms_keys) == 0:
raise click.UsageError('No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
options = ['{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys]
kms_key = choice(prompt='Please select the encryption key', options=options)
kms_keyid = kms_key.split(':')[0]
variables['kms_arn'] = [k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid][0]
for key in [k for k in variables if k.startswith('pgpassword_')]:
encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
set_default_variables(variables)
sg_name = 'app-spilo'
rules_missing = check_security_group(sg_name,
[('tcp', 22), ('tcp', POSTGRES_PORT), ('tcp', HEALTHCHECK_PORT)],
region, allow_from_self=True)
if ('tcp', 22) in rules_missing:
warning('Security group {} does not allow SSH access, you will not be able to ssh into your servers'.
format(sg_name))
if ('tcp', POSTGRES_PORT) in rules_missing:
error('Security group {} does not allow inbound TCP traffic on the default postgres port ({})'.format(
sg_name, POSTGRES_PORT
))
if ('tcp', HEALTHCHECK_PORT) in rules_missing:
error('Security group {} does not allow inbound TCP traffic on the default health check port ({})'.
format(sg_name, HEALTHCHECK_PORT))
variables['spilo_sg_id'] = get_security_group(region, sg_name).id
check_s3_bucket(variables['wal_s3_bucket'], region)
return variables
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm("Do you want to set the docker image now? [No]"):
prompt(variables, "docker_image", "Docker Image Version", default=get_latest_spilo_image())
prompt(
variables,
"wal_s3_bucket",
"Postgres WAL S3 bucket to use",
default="{}-{}-spilo-app".format(get_account_alias(), region),
)
prompt(variables, "instance_type", "EC2 instance type", default="t2.micro")
variables["hosted_zone"] = account_info.Domain or defaults["hosted_zone"]
if variables["hosted_zone"][-1:] != ".":
variables["hosted_zone"] += "."
prompt(variables, "discovery_domain", "ETCD Discovery Domain", default="postgres." + variables["hosted_zone"][:-1])
variables["add_replica_loadbalancer"] = click.confirm("Do you want a replica ELB?", default=False)
prompt(
variables,
"elb_access_cidr",
"Which network should be allowed to access the ELB" "s? (default=vpc)",
default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute="cidr_block"),
)
odd_sg_name = "Odd (SSH Bastion Host)"
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm(
"Do you want to allow access to the Spilo nodes from {}?".format(odd_sg_name), default=True
):
variables["odd_sg_id"] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client("ec2", region)
filters = [{"Name": "tag-key", "Values": ["StackName"]}, {"Name": "tag-value", "Values": ["zmon-worker"]}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get("Reservations", []):
for instance in reservation.get("Instances", []):
zmon_sgs += [sg["GroupId"] for sg in instance.get("SecurityGroups", []) if "zmon" in sg["GroupName"]]
if len(zmon_sgs) == 0:
warning("Could not find zmon security group")
else:
click.confirm("Do you want to allow access to the Spilo nodes from zmon?", default=True)
if len(zmon_sgs) > 1:
prompt(variables, "zmon_sg_id", "Which Security Group should we allow access from? {}".format(zmon_sgs))
else:
variables["zmon_sg_id"] = zmon_sgs[0]
if variables["instance_type"].lower().split(".")[0] in ("c3", "g2", "hi1", "i2", "m3", "r3"):
variables["use_ebs"] = click.confirm(
"Do you want database data directory on external (EBS) storage? [Yes]", default=defaults["use_ebs"]
)
else:
variables["use_ebs"] = True
if variables["use_ebs"]:
prompt(variables, "volume_size", "Database volume size (GB, 10 or more)", default=defaults["volume_size"])
prompt(variables, "volume_type", "Database volume type (gp2, io1 or standard)", default=defaults["volume_type"])
if variables["volume_type"] == "io1":
pio_max = variables["volume_size"] * 30
prompt(
variables,
"volume_iops",
"Provisioned I/O operations per second (100 - {0})".format(pio_max),
default=str(pio_max),
)
prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
if ebs_optimized_supported(variables["instance_type"]):
variables["ebs_optimized"] = True
prompt(variables, "fstype", "Filesystem for the data partition", default=defaults["fstype"])
prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)", default=defaults["fsoptions"])
prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
prompt(
variables,
"pgpassword_superuser",
"Password for PostgreSQL superuser [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True,
)
prompt(
variables,
"pgpassword_standby",
"Password for PostgreSQL user standby [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True,
)
prompt(
variables,
"pgpassword_admin",
"Password for PostgreSQL user admin",
show_default=True,
default=defaults["pgpassword_admin"],
hide_input=True,
confirmation_prompt=True,
)
if click.confirm("Do you wish to encrypt these passwords using KMS?", default=False):
kms_keys = [k for k in list_kms_keys(region) if "alias/aws/ebs" not in k["aliases"]]
if len(kms_keys) == 0:
raise click.UsageError(
"No KMS key is available for encrypting and decrypting. " "Ensure you have at least 1 key available."
)
options = ["{}: {}".format(k["KeyId"], k["Description"]) for k in kms_keys]
kms_key = choice(prompt="Please select the encryption key", options=options)
kms_keyid = kms_key.split(":")[0]
variables["kms_arn"] = [k["Arn"] for k in kms_keys if k["KeyId"] == kms_keyid][0]
for key in [k for k in variables if k.startswith("pgpassword_")]:
encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = "aws:kms:{}".format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables["wal_s3_bucket"], region)
return variables
def gather_user_variables(variables, region, account_info):
# maximal 32 characters because of the loadbalancer-name
prompt(variables,
'application_id',
'Application ID',
default='hello-world',
value_proc=check_value(60, '^[a-zA-Z][-a-zA-Z0-9]*$'))
prompt(
variables,
'docker_image',
'Docker image without tag/version (e.g. "pierone.example.org/myteam/myapp")',
default='stups/hello-world')
prompt(variables, 'http_port', 'HTTP port', default=8080, type=int)
prompt(variables,
'http_health_check_path',
'HTTP health check path',
default='/')
prompt(variables, 'instance_type', 'EC2 instance type', default='t2.micro')
if 'pierone' in variables['docker_image'] or confirm(
'Did you need OAuth-Credentials from Mint?'):
prompt(variables,
'mint_bucket',
'Mint S3 bucket name',
default=lambda: get_mint_bucket_name(region))
else:
variables['mint_bucket'] = None
variables['loadbalancer_scheme'] = choice(
prompt='Please select the load balancer scheme',
options=[('internal', 'internal: only accessible from the own VPC'),
('internet-facing',
'internet-facing: accessible from the public internet')],
default='internal')
http_port = variables['http_port']
sg_name = 'app-{}'.format(variables['application_id'])
rules_missing = check_security_group(sg_name, [('tcp', 22),
('tcp', http_port)],
region,
allow_from_self=True)
if ('tcp', 22) in rules_missing:
warning(
'Security group {} does not allow SSH access, you will not be able to ssh into your servers'
.format(sg_name))
if ('tcp', http_port) in rules_missing:
error(
'Security group {} does not allow inbound TCP traffic on the specified HTTP port ({})'
.format(sg_name, http_port))
rules_missing = check_security_group(sg_name + '-lb', [('tcp', 443)],
region)
if rules_missing:
error(
'Load balancer security group {} does not allow inbound HTTPS traffic'
.format(sg_name))
check_iam_role(variables['application_id'], variables['mint_bucket'],
region)
return variables
#!/usr/bin/python3
from clickclick import choice
from subprocess import call
import pathlib
USR_BIN = pathlib.Path('/usr/bin')
cwd = pathlib.Path.cwd()
project_name = cwd.parts[-1]
print("Creating venv for", project_name)
python_binaries = sorted(list(set(USR_BIN.glob('python*.*')) - set(USR_BIN.glob('*m')) - set(USR_BIN.glob('*-*'))))
selected_binary = choice("Which Python Version Do You Want To Use?", python_binaries, USR_BIN / "python3.4")
version = str(selected_binary)[-3:]
# TODO make path configurable
venv_name = "{project_name}-{version}".format_map(locals())
print("$ virtualenv --verbose -p /usr/bin/python{version} ~/virtual_environments/{venv_name}".format_map(locals()))
call("virtualenv --verbose -p {selected_binary} ~/virtual_environments/{venv_name}".format_map(locals()), shell=True)
print("source /home/joaosantos/virtual_environments/{venv_name}/bin/activate".format_map(locals()))
activate_zsh = cwd / '.autoenv.zsh'
with activate_zsh.open('w+') as script:
script.write("source /home/joaosantos/virtual_environments/{venv_name}/bin/activate".format_map(locals()))
deactivate_zsh = cwd / '.autoenv_leave.zsh'
def choicetest():
print('>>{}<<'.format(choice('Please choose', ['a', 'b', 'c'], default='c')))
print('>>{}<<'.format(choice('Please choose', [('a', 'Label A'), ('b', 'Label B'), ('c', 'Label C')], default='b')))
print('>>{}<<'.format(choice('Please choose', [('a', 'Label A'), ('b', 'Label B'), ('c', 'Label C')], default='x')))
def saml_login(profile, region, url, user, password=None, role=None, print_env_vars=False,
overwrite_default_credentials=False):
session = requests.Session()
response = session.get(url)
keyring_key = 'aws-minion.saml'
password = password or keyring.get_password(keyring_key, user)
if not password:
password = click.prompt('Password', hide_input=True)
with Action('Authenticating against {url}..', **vars()) as act:
# NOTE: parameters are hardcoded for Shibboleth IDP
data = {'j_username': user, 'j_password': password, 'submit': 'Login'}
response2 = session.post(response.url, data=data)
saml_xml = get_saml_response(response2.text)
if not saml_xml:
act.error('LOGIN FAILED')
click.secho('SAML login with user "{}" failed, please check your username and password.\n'.format(user) +
'You might need to change the password in your keyring (e.g. Mac OS X keychain) ' +
'or use the "--password" option.', bold=True, fg='blue')
return
url = get_form_action(response2.text)
encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64')
response3 = session.post(url, data={'SAMLResponse': encoded_xml})
account_names = get_account_names(response3.text)
keyring.set_password(keyring_key, user, password)
with Action('Checking SAML roles..') as act:
roles = get_roles(saml_xml)
if not roles:
act.error('NO VALID ROLE FOUND')
return
if len(roles) == 1:
provider_arn, role_arn = roles[0]
elif role:
matching_roles = [_role for _role in roles if role in get_role_label(_role, account_names)]
if not matching_roles or len(matching_roles) > 1:
raise click.UsageError('Given role (--role) was not found or not unique')
provider_arn, role_arn = matching_roles[0]
else:
roles.sort()
provider_arn, role_arn = choice('Multiple roles found, please select one.',
[(r, get_role_label(r, account_names)) for r in roles])
with Action('Assuming role "{role_label}"..', role_label=get_role_label((provider_arn, role_arn), account_names)):
saml_assertion = codecs.encode(saml_xml.encode('utf-8'), 'base64').decode('ascii').replace('\n', '')
# botocore NEEDS some credentials, but does not care about their actual values
os.environ['AWS_ACCESS_KEY_ID'] = 'fake123'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123'
try:
session = botocore.session.get_session()
sts = session.get_service('sts')
operation = sts.get_operation('AssumeRoleWithSAML')
endpoint = sts.get_endpoint(region)
endpoint._signature_version = None
http_response, response_data = operation.call(endpoint, role_arn=role_arn, principal_arn=provider_arn,
SAMLAssertion=saml_assertion)
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
key_id = response_data['Credentials']['AccessKeyId']
secret = response_data['Credentials']['SecretAccessKey']
session_token = response_data['Credentials']['SessionToken']
if print_env_vars:
# different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both
click.secho(dedent('''\
# environment variables with temporary AWS credentials:
export AWS_ACCESS_KEY_ID="{key_id}"
export AWS_SECRET_ACCESS_KEY="{secret}"
export AWS_SESSION_TOKEN="{session_token}")
export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()), fg='blue')
profiles_to_write = set([profile])
if overwrite_default_credentials:
profiles_to_write.add('default')
with Action('Writing temporary AWS credentials..'):
for prof in profiles_to_write:
write_aws_credentials(prof, key_id, secret, session_token)
def saml_login(profile,
region,
url,
user,
password=None,
role=None,
print_env_vars=False,
overwrite_default_credentials=False):
session = requests.Session()
response = session.get(url)
keyring_key = 'aws-minion.saml'
password = password or keyring.get_password(keyring_key, user)
if not password:
password = click.prompt('Password', hide_input=True)
with Action('Authenticating against {url}..', **vars()) as act:
# NOTE: parameters are hardcoded for Shibboleth IDP
data = {'j_username': user, 'j_password': password, 'submit': 'Login'}
response2 = session.post(response.url, data=data)
saml_xml = get_saml_response(response2.text)
if not saml_xml:
act.error('LOGIN FAILED')
click.secho(
'SAML login with user "{}" failed, please check your username and password.\n'
.format(user) +
'You might need to change the password in your keyring (e.g. Mac OS X keychain) '
+ 'or use the "--password" option.',
bold=True,
fg='blue')
return
url = get_form_action(response2.text)
encoded_xml = codecs.encode(saml_xml.encode('utf-8'), 'base64')
response3 = session.post(url, data={'SAMLResponse': encoded_xml})
account_names = get_account_names(response3.text)
keyring.set_password(keyring_key, user, password)
with Action('Checking SAML roles..') as act:
roles = get_roles(saml_xml)
if not roles:
act.error('NO VALID ROLE FOUND')
return
if len(roles) == 1:
provider_arn, role_arn = roles[0]
elif role:
matching_roles = [
_role for _role in roles
if role in get_role_label(_role, account_names)
]
if not matching_roles or len(matching_roles) > 1:
raise click.UsageError(
'Given role (--role) was not found or not unique')
provider_arn, role_arn = matching_roles[0]
else:
roles.sort()
provider_arn, role_arn = choice(
'Multiple roles found, please select one.',
[(r, get_role_label(r, account_names)) for r in roles])
with Action('Assuming role "{role_label}"..',
role_label=get_role_label((provider_arn, role_arn),
account_names)):
saml_assertion = codecs.encode(saml_xml.encode('utf-8'),
'base64').decode('ascii').replace(
'\n', '')
# botocore NEEDS some credentials, but does not care about their actual values
os.environ['AWS_ACCESS_KEY_ID'] = 'fake123'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'fake123'
try:
session = botocore.session.get_session()
sts = session.get_service('sts')
operation = sts.get_operation('AssumeRoleWithSAML')
endpoint = sts.get_endpoint(region)
endpoint._signature_version = None
http_response, response_data = operation.call(
endpoint,
role_arn=role_arn,
principal_arn=provider_arn,
SAMLAssertion=saml_assertion)
finally:
del os.environ['AWS_ACCESS_KEY_ID']
del os.environ['AWS_SECRET_ACCESS_KEY']
key_id = response_data['Credentials']['AccessKeyId']
secret = response_data['Credentials']['SecretAccessKey']
session_token = response_data['Credentials']['SessionToken']
if print_env_vars:
# different AWS SDKs expect either AWS_SESSION_TOKEN or AWS_SECURITY_TOKEN, so set both
click.secho(dedent('''\
# environment variables with temporary AWS credentials:
export AWS_ACCESS_KEY_ID="{key_id}"
export AWS_SECRET_ACCESS_KEY="{secret}"
export AWS_SESSION_TOKEN="{session_token}")
export AWS_SECURITY_TOKEN="{session_token}"''').format(**vars()),
fg='blue')
profiles_to_write = set([profile])
if overwrite_default_credentials:
profiles_to_write.add('default')
with Action('Writing temporary AWS credentials..'):
for prof in profiles_to_write:
write_aws_credentials(prof, key_id, secret, session_token)
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm('Do you want to set the docker image now? [No]'):
prompt(variables,
"docker_image",
"Docker Image Version",
default=get_latest_image())
prompt(variables,
'wal_s3_bucket',
'Postgres WAL S3 bucket to use',
default='{}-{}-spilo-app'.format(get_account_alias(), region))
prompt(variables,
'instance_type',
'EC2 instance type',
default='t2.medium')
variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
if (variables['hosted_zone'][-1:] != '.'):
variables['hosted_zone'] += '.'
prompt(variables,
'discovery_domain',
'ETCD Discovery Domain',
default='postgres.' + variables['hosted_zone'][:-1])
variables['add_replica_loadbalancer'] = click.confirm(
'Do you want a replica ELB?', default=False)
prompt(variables,
'elb_access_cidr',
'Which network should be allowed to access the ELB'
's? (default=vpc)',
default=get_vpc_attribute(region=region,
vpc_id=account_info.VpcID,
attribute='cidr_block'))
odd_sg_name = 'Odd (SSH Bastion Host)'
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm(
'Do you want to allow access to the Spilo nodes from {}?'.format(
odd_sg_name),
default=True):
variables['odd_sg_id'] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client('ec2', region)
filters = [{
'Name': 'tag-key',
'Values': ['StackName']
}, {
'Name': 'tag-value',
'Values': ['zmon-worker']
}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get(
'Reservations', []):
for instance in reservation.get('Instances', []):
zmon_sgs += [
sg['GroupId'] for sg in instance.get('SecurityGroups', [])
if 'zmon' in sg['GroupName']
]
if len(zmon_sgs) == 0:
warning('Could not find zmon security group')
else:
click.confirm(
'Do you want to allow access to the Spilo nodes from zmon?',
default=True)
if len(zmon_sgs) > 1:
prompt(
variables, 'zmon_sg_id',
'Which Security Group should we allow access from? {}'.format(
zmon_sgs))
else:
variables['zmon_sg_id'] = zmon_sgs[0]
if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1',
'i2', 'm3', 'r3'):
variables['use_ebs'] = click.confirm(
'Do you want database data directory on external (EBS) storage? [Yes]',
default=defaults['use_ebs'])
else:
variables['use_ebs'] = True
if variables['use_ebs']:
prompt(variables,
'volume_size',
'Database volume size (GB, 10 or more)',
default=defaults['volume_size'])
prompt(variables,
'volume_type',
'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
pio_max = variables['volume_size'] * 30
prompt(variables,
"volume_iops",
'Provisioned I/O operations per second (100 - {0})'.format(
pio_max),
default=str(pio_max))
prompt(variables,
"snapshot_id",
"ID of the snapshot to populate EBS volume from",
default="")
if ebs_optimized_supported(variables['instance_type']):
variables['ebs_optimized'] = True
prompt(variables,
"fstype",
"Filesystem for the data partition",
default=defaults['fstype'])
prompt(variables,
"fsoptions",
"Filesystem mount options (comma-separated)",
default=defaults['fsoptions'])
prompt(variables, "scalyr_account_key",
"Account key for your scalyr account", "")
prompt(variables,
'pgpassword_superuser',
"Password for PostgreSQL superuser [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True)
prompt(variables,
'pgpassword_standby',
"Password for PostgreSQL user standby [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True)
prompt(variables,
'pgpassword_admin',
"Password for PostgreSQL user admin",
show_default=True,
default=defaults['pgpassword_admin'],
hide_input=True,
confirmation_prompt=True)
if click.confirm('Do you wish to encrypt these passwords using KMS?',
default=False):
kms_keys = [
k for k in list_kms_keys(region)
if 'alias/aws/ebs' not in k['aliases']
]
if len(kms_keys) == 0:
raise click.UsageError(
'No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
options = [
'{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys
]
kms_key = choice(prompt='Please select the encryption key',
options=options)
kms_keyid = kms_key.split(':')[0]
variables['kms_arn'] = [
k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid
][0]
for key in [
k for k in variables
if k.startswith('pgpassword_') or k == 'scalyr_account_key'
]:
encrypted = encrypt(region=region,
KeyId=kms_keyid,
Plaintext=variables[key],
b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables['wal_s3_bucket'], region)
return variables
def work_done(percentage):
'''Work done in ?? %'''
state = choice('Please select the state of your work', ['Done', 'In Progress', 'unknown', 'lost'], default='lost')
print('Your work is {}% {}'.format(percentage, state))
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm('Do you want to set the docker image now? [No]'):
prompt(variables, "docker_image", "Docker Image Version", default=get_latest_image())
prompt(variables, 'wal_s3_bucket', 'Postgres WAL S3 bucket to use',
default='{}-{}-spilo-app'.format(get_account_alias(), region))
prompt(variables, 'instance_type', 'EC2 instance type', default='t2.medium')
variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
if (variables['hosted_zone'][-1:] != '.'):
variables['hosted_zone'] += '.'
prompt(variables, 'discovery_domain', 'ETCD Discovery Domain',
default='postgres.' + variables['hosted_zone'][:-1])
variables['add_replica_loadbalancer'] = click.confirm('Do you want a replica ELB?', default=False)
prompt(variables, 'elb_access_cidr', 'Which network should be allowed to access the ELB''s? (default=vpc)',
default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute='cidr_block'))
odd_sg_name = 'Odd (SSH Bastion Host)'
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm('Do you want to allow access to the Spilo nodes from {}?'.format(odd_sg_name),
default=True):
variables['odd_sg_id'] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client('ec2', region)
filters = [{'Name': 'tag-key', 'Values': ['StackName']}, {'Name': 'tag-value', 'Values': ['zmon-appliance']}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get('Reservations', []):
for instance in reservation.get('Instances', []):
zmon_sgs += [sg['GroupId'] for sg in instance.get('SecurityGroups', []) if 'zmon' in sg['GroupName']]
if len(zmon_sgs) == 0:
warning('Could not find zmon security group, do you have the zmon-appliance deployed?')
else:
click.confirm('Do you want to allow access to the Spilo nodes from zmon?', default=True)
if len(zmon_sgs) > 1:
prompt(variables, 'zmon_sg_id', 'Which Security Group should we allow access from? {}'.format(zmon_sgs))
else:
variables['zmon_sg_id'] = zmon_sgs[0]
if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1', 'i2', 'm3', 'r3'):
variables['use_ebs'] = click.confirm('Do you want database data directory on external (EBS) storage? [Yes]',
default=defaults['use_ebs'])
else:
variables['use_ebs'] = True
if variables['use_ebs']:
prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
pio_max = variables['volume_size'] * 30
prompt(variables, "volume_iops", 'Provisioned I/O operations per second (100 - {0})'.
format(pio_max), default=str(pio_max))
prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
if ebs_optimized_supported(variables['instance_type']):
variables['ebs_optimized'] = True
prompt(variables, "fstype", "Filesystem for the data partition", default=defaults['fstype'])
prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)",
default=defaults['fsoptions'])
prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
prompt(variables, 'pgpassword_superuser', "Password for PostgreSQL superuser [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_standby', "Password for PostgreSQL user standby [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_admin', "Password for PostgreSQL user admin", show_default=True,
default=defaults['pgpassword_admin'], hide_input=True, confirmation_prompt=True)
if click.confirm('Do you wish to encrypt these passwords using KMS?', default=False):
kms_keys = [k for k in list_kms_keys(region) if 'alias/aws/ebs' not in k['aliases']]
if len(kms_keys) == 0:
raise click.UsageError('No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
options = ['{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys]
kms_key = choice(prompt='Please select the encryption key', options=options)
kms_keyid = kms_key.split(':')[0]
variables['kms_arn'] = [k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid][0]
for key in [k for k in variables if k.startswith('pgpassword_') or k == 'scalyr_account_key']:
if variables[key]:
encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables['wal_s3_bucket'], region)
return variables
