def test_get_vpc_attribute(monkeypatch):
from collections import namedtuple
ec2 = MagicMock()
ec2.Vpc.return_value = namedtuple('a', 'VpcId')('dummy')
monkeypatch.setattr('boto3.resource', MagicMock(return_value=ec2))
assert get_vpc_attribute('r', 'a', 'VpcId') == 'dummy'
assert get_vpc_attribute('r', 'a', 'nonexistent') is None
def test_get_vpc_attribute(monkeypatch):
from collections import namedtuple
ec2 = MagicMock()
ec2.Vpc.return_value = namedtuple("a", "VpcId")("dummy")
boto3 = MagicMock()
monkeypatch.setattr("boto3.resource", MagicMock(return_value=ec2))
assert get_vpc_attribute("r", "a", "VpcId") == "dummy"
assert get_vpc_attribute("r", "a", "nonexistent") is None
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm('Do you want to set the docker image now? [No]'):
prompt(variables, "docker_image", "Docker Image Version", default=get_latest_image())
prompt(variables, 'wal_s3_bucket', 'Postgres WAL S3 bucket to use',
default='{}-{}-spilo-app'.format(get_account_alias(), region))
prompt(variables, 'instance_type', 'EC2 instance type', default='t2.medium')
variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
if (variables['hosted_zone'][-1:] != '.'):
variables['hosted_zone'] += '.'
prompt(variables, 'discovery_domain', 'ETCD Discovery Domain',
default='postgres.' + variables['hosted_zone'][:-1])
variables['add_replica_loadbalancer'] = click.confirm('Do you want a replica ELB?', default=False)
prompt(variables, 'elb_access_cidr', 'Which network should be allowed to access the ELB''s? (default=vpc)',
default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute='cidr_block'))
odd_sg_name = 'Odd (SSH Bastion Host)'
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm('Do you want to allow access to the Spilo nodes from {}?'.format(odd_sg_name),
default=True):
variables['odd_sg_id'] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client('ec2', region)
filters = [{'Name': 'tag-key', 'Values': ['StackName']}, {'Name': 'tag-value', 'Values': ['zmon-appliance']}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get('Reservations', []):
for instance in reservation.get('Instances', []):
zmon_sgs += [sg['GroupId'] for sg in instance.get('SecurityGroups', []) if 'zmon' in sg['GroupName']]
if len(zmon_sgs) == 0:
warning('Could not find zmon security group, do you have the zmon-appliance deployed?')
else:
click.confirm('Do you want to allow access to the Spilo nodes from zmon?', default=True)
if len(zmon_sgs) > 1:
prompt(variables, 'zmon_sg_id', 'Which Security Group should we allow access from? {}'.format(zmon_sgs))
else:
variables['zmon_sg_id'] = zmon_sgs[0]
if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1', 'i2', 'm3', 'r3'):
variables['use_ebs'] = click.confirm('Do you want database data directory on external (EBS) storage? [Yes]',
default=defaults['use_ebs'])
else:
variables['use_ebs'] = True
if variables['use_ebs']:
prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
pio_max = variables['volume_size'] * 30
prompt(variables, "volume_iops", 'Provisioned I/O operations per second (100 - {0})'.
format(pio_max), default=str(pio_max))
prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
if ebs_optimized_supported(variables['instance_type']):
variables['ebs_optimized'] = True
prompt(variables, "fstype", "Filesystem for the data partition", default=defaults['fstype'])
prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)",
default=defaults['fsoptions'])
prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
prompt(variables, 'pgpassword_superuser', "Password for PostgreSQL superuser [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_standby', "Password for PostgreSQL user standby [random]", show_default=False,
default=generate_random_password, hide_input=True, confirmation_prompt=True)
prompt(variables, 'pgpassword_admin', "Password for PostgreSQL user admin", show_default=True,
default=defaults['pgpassword_admin'], hide_input=True, confirmation_prompt=True)
if click.confirm('Do you wish to encrypt these passwords using KMS?', default=False):
kms_keys = [k for k in list_kms_keys(region) if 'alias/aws/ebs' not in k['aliases']]
if len(kms_keys) == 0:
raise click.UsageError('No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
options = ['{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys]
kms_key = choice(prompt='Please select the encryption key', options=options)
kms_keyid = kms_key.split(':')[0]
variables['kms_arn'] = [k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid][0]
for key in [k for k in variables if k.startswith('pgpassword_') or k == 'scalyr_account_key']:
if variables[key]:
encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables['wal_s3_bucket'], region)
return variables
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm('Do you want to set the docker image now? [No]'):
prompt(variables,
"docker_image",
"Docker Image Version",
default=get_latest_image())
prompt(variables,
'wal_s3_bucket',
'Postgres WAL S3 bucket to use',
default='{}-{}-spilo-app'.format(get_account_alias(), region))
prompt(variables,
'instance_type',
'EC2 instance type',
default='t2.medium')
variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
if (variables['hosted_zone'][-1:] != '.'):
variables['hosted_zone'] += '.'
prompt(variables,
'discovery_domain',
'ETCD Discovery Domain',
default='postgres.' + variables['hosted_zone'][:-1])
variables['add_replica_loadbalancer'] = click.confirm(
'Do you want a replica ELB?', default=False)
prompt(variables,
'elb_access_cidr',
'Which network should be allowed to access the ELB'
's? (default=vpc)',
default=get_vpc_attribute(region=region,
vpc_id=account_info.VpcID,
attribute='cidr_block'))
odd_sg_name = 'Odd (SSH Bastion Host)'
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm(
'Do you want to allow access to the Spilo nodes from {}?'.format(
odd_sg_name),
default=True):
variables['odd_sg_id'] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client('ec2', region)
filters = [{
'Name': 'tag-key',
'Values': ['StackName']
}, {
'Name': 'tag-value',
'Values': ['zmon-worker']
}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get(
'Reservations', []):
for instance in reservation.get('Instances', []):
zmon_sgs += [
sg['GroupId'] for sg in instance.get('SecurityGroups', [])
if 'zmon' in sg['GroupName']
]
if len(zmon_sgs) == 0:
warning('Could not find zmon security group')
else:
click.confirm(
'Do you want to allow access to the Spilo nodes from zmon?',
default=True)
if len(zmon_sgs) > 1:
prompt(
variables, 'zmon_sg_id',
'Which Security Group should we allow access from? {}'.format(
zmon_sgs))
else:
variables['zmon_sg_id'] = zmon_sgs[0]
if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1',
'i2', 'm3', 'r3'):
variables['use_ebs'] = click.confirm(
'Do you want database data directory on external (EBS) storage? [Yes]',
default=defaults['use_ebs'])
else:
variables['use_ebs'] = True
if variables['use_ebs']:
prompt(variables,
'volume_size',
'Database volume size (GB, 10 or more)',
default=defaults['volume_size'])
prompt(variables,
'volume_type',
'Database volume type (gp2, io1 or standard)',
default=defaults['volume_type'])
if variables['volume_type'] == 'io1':
pio_max = variables['volume_size'] * 30
prompt(variables,
"volume_iops",
'Provisioned I/O operations per second (100 - {0})'.format(
pio_max),
default=str(pio_max))
prompt(variables,
"snapshot_id",
"ID of the snapshot to populate EBS volume from",
default="")
if ebs_optimized_supported(variables['instance_type']):
variables['ebs_optimized'] = True
prompt(variables,
"fstype",
"Filesystem for the data partition",
default=defaults['fstype'])
prompt(variables,
"fsoptions",
"Filesystem mount options (comma-separated)",
default=defaults['fsoptions'])
prompt(variables, "scalyr_account_key",
"Account key for your scalyr account", "")
prompt(variables,
'pgpassword_superuser',
"Password for PostgreSQL superuser [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True)
prompt(variables,
'pgpassword_standby',
"Password for PostgreSQL user standby [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True)
prompt(variables,
'pgpassword_admin',
"Password for PostgreSQL user admin",
show_default=True,
default=defaults['pgpassword_admin'],
hide_input=True,
confirmation_prompt=True)
if click.confirm('Do you wish to encrypt these passwords using KMS?',
default=False):
kms_keys = [
k for k in list_kms_keys(region)
if 'alias/aws/ebs' not in k['aliases']
]
if len(kms_keys) == 0:
raise click.UsageError(
'No KMS key is available for encrypting and decrypting. '
'Ensure you have at least 1 key available.')
options = [
'{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys
]
kms_key = choice(prompt='Please select the encryption key',
options=options)
kms_keyid = kms_key.split(':')[0]
variables['kms_arn'] = [
k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid
][0]
for key in [
k for k in variables
if k.startswith('pgpassword_') or k == 'scalyr_account_key'
]:
encrypted = encrypt(region=region,
KeyId=kms_keyid,
Plaintext=variables[key],
b64encode=True)
variables[key] = 'aws:kms:{}'.format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables['wal_s3_bucket'], region)
return variables
def gather_user_variables(variables, region, account_info):
defaults = set_default_variables(dict())
if click.confirm("Do you want to set the docker image now? [No]"):
prompt(variables, "docker_image", "Docker Image Version", default=get_latest_spilo_image())
prompt(
variables,
"wal_s3_bucket",
"Postgres WAL S3 bucket to use",
default="{}-{}-spilo-app".format(get_account_alias(), region),
)
prompt(variables, "instance_type", "EC2 instance type", default="t2.micro")
variables["hosted_zone"] = account_info.Domain or defaults["hosted_zone"]
if variables["hosted_zone"][-1:] != ".":
variables["hosted_zone"] += "."
prompt(variables, "discovery_domain", "ETCD Discovery Domain", default="postgres." + variables["hosted_zone"][:-1])
variables["add_replica_loadbalancer"] = click.confirm("Do you want a replica ELB?", default=False)
prompt(
variables,
"elb_access_cidr",
"Which network should be allowed to access the ELB" "s? (default=vpc)",
default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute="cidr_block"),
)
odd_sg_name = "Odd (SSH Bastion Host)"
odd_sg = get_security_group(region, odd_sg_name)
if odd_sg and click.confirm(
"Do you want to allow access to the Spilo nodes from {}?".format(odd_sg_name), default=True
):
variables["odd_sg_id"] = odd_sg.group_id
# Find all Security Groups attached to the zmon worker with 'zmon' in their name
ec2 = boto3.client("ec2", region)
filters = [{"Name": "tag-key", "Values": ["StackName"]}, {"Name": "tag-value", "Values": ["zmon-worker"]}]
zmon_sgs = list()
for reservation in ec2.describe_instances(Filters=filters).get("Reservations", []):
for instance in reservation.get("Instances", []):
zmon_sgs += [sg["GroupId"] for sg in instance.get("SecurityGroups", []) if "zmon" in sg["GroupName"]]
if len(zmon_sgs) == 0:
warning("Could not find zmon security group")
else:
click.confirm("Do you want to allow access to the Spilo nodes from zmon?", default=True)
if len(zmon_sgs) > 1:
prompt(variables, "zmon_sg_id", "Which Security Group should we allow access from? {}".format(zmon_sgs))
else:
variables["zmon_sg_id"] = zmon_sgs[0]
if variables["instance_type"].lower().split(".")[0] in ("c3", "g2", "hi1", "i2", "m3", "r3"):
variables["use_ebs"] = click.confirm(
"Do you want database data directory on external (EBS) storage? [Yes]", default=defaults["use_ebs"]
)
else:
variables["use_ebs"] = True
if variables["use_ebs"]:
prompt(variables, "volume_size", "Database volume size (GB, 10 or more)", default=defaults["volume_size"])
prompt(variables, "volume_type", "Database volume type (gp2, io1 or standard)", default=defaults["volume_type"])
if variables["volume_type"] == "io1":
pio_max = variables["volume_size"] * 30
prompt(
variables,
"volume_iops",
"Provisioned I/O operations per second (100 - {0})".format(pio_max),
default=str(pio_max),
)
prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
if ebs_optimized_supported(variables["instance_type"]):
variables["ebs_optimized"] = True
prompt(variables, "fstype", "Filesystem for the data partition", default=defaults["fstype"])
prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)", default=defaults["fsoptions"])
prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")
prompt(
variables,
"pgpassword_superuser",
"Password for PostgreSQL superuser [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True,
)
prompt(
variables,
"pgpassword_standby",
"Password for PostgreSQL user standby [random]",
show_default=False,
default=generate_random_password,
hide_input=True,
confirmation_prompt=True,
)
prompt(
variables,
"pgpassword_admin",
"Password for PostgreSQL user admin",
show_default=True,
default=defaults["pgpassword_admin"],
hide_input=True,
confirmation_prompt=True,
)
if click.confirm("Do you wish to encrypt these passwords using KMS?", default=False):
kms_keys = [k for k in list_kms_keys(region) if "alias/aws/ebs" not in k["aliases"]]
if len(kms_keys) == 0:
raise click.UsageError(
"No KMS key is available for encrypting and decrypting. " "Ensure you have at least 1 key available."
)
options = ["{}: {}".format(k["KeyId"], k["Description"]) for k in kms_keys]
kms_key = choice(prompt="Please select the encryption key", options=options)
kms_keyid = kms_key.split(":")[0]
variables["kms_arn"] = [k["Arn"] for k in kms_keys if k["KeyId"] == kms_keyid][0]
for key in [k for k in variables if k.startswith("pgpassword_")]:
encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
variables[key] = "aws:kms:{}".format(encrypted)
set_default_variables(variables)
check_s3_bucket(variables["wal_s3_bucket"], region)
return variables