Пример #1
0
def test_get_vpc_attribute(monkeypatch):
    from collections import namedtuple

    ec2 = MagicMock()
    ec2.Vpc.return_value = namedtuple('a', 'VpcId')('dummy')

    monkeypatch.setattr('boto3.resource', MagicMock(return_value=ec2))

    assert get_vpc_attribute('r', 'a', 'VpcId') == 'dummy'
    assert get_vpc_attribute('r', 'a', 'nonexistent') is None
Пример #2
0
def test_get_vpc_attribute(monkeypatch):
    from collections import namedtuple

    ec2 = MagicMock()
    ec2.Vpc.return_value = namedtuple("a", "VpcId")("dummy")

    boto3 = MagicMock()
    monkeypatch.setattr("boto3.resource", MagicMock(return_value=ec2))

    assert get_vpc_attribute("r", "a", "VpcId") == "dummy"
    assert get_vpc_attribute("r", "a", "nonexistent") is None
Пример #3
0
def gather_user_variables(variables, region, account_info):
    defaults = set_default_variables(dict())

    if click.confirm('Do you want to set the docker image now? [No]'):
        prompt(variables, "docker_image", "Docker Image Version", default=get_latest_image())

    prompt(variables, 'wal_s3_bucket', 'Postgres WAL S3 bucket to use',
           default='{}-{}-spilo-app'.format(get_account_alias(), region))

    prompt(variables, 'instance_type', 'EC2 instance type', default='t2.medium')

    variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
    if (variables['hosted_zone'][-1:] != '.'):
        variables['hosted_zone'] += '.'
    prompt(variables, 'discovery_domain', 'ETCD Discovery Domain',
           default='postgres.' + variables['hosted_zone'][:-1])

    variables['add_replica_loadbalancer'] = click.confirm('Do you want a replica ELB?', default=False)

    prompt(variables, 'elb_access_cidr', 'Which network should be allowed to access the ELB''s? (default=vpc)',
           default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute='cidr_block'))

    odd_sg_name = 'Odd (SSH Bastion Host)'
    odd_sg = get_security_group(region, odd_sg_name)
    if odd_sg and click.confirm('Do you want to allow access to the Spilo nodes from {}?'.format(odd_sg_name),
                                default=True):
        variables['odd_sg_id'] = odd_sg.group_id

    # Find all Security Groups attached to the zmon worker with 'zmon' in their name
    ec2 = boto3.client('ec2', region)
    filters = [{'Name': 'tag-key', 'Values': ['StackName']}, {'Name': 'tag-value', 'Values': ['zmon-appliance']}]
    zmon_sgs = list()
    for reservation in ec2.describe_instances(Filters=filters).get('Reservations', []):
        for instance in reservation.get('Instances', []):
            zmon_sgs += [sg['GroupId'] for sg in instance.get('SecurityGroups', []) if 'zmon' in sg['GroupName']]

    if len(zmon_sgs) == 0:
        warning('Could not find zmon security group, do you have the zmon-appliance deployed?')
    else:
        click.confirm('Do you want to allow access to the Spilo nodes from zmon?', default=True)
        if len(zmon_sgs) > 1:
            prompt(variables, 'zmon_sg_id', 'Which Security Group should we allow access from? {}'.format(zmon_sgs))
        else:
            variables['zmon_sg_id'] = zmon_sgs[0]

    if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1', 'i2', 'm3', 'r3'):
        variables['use_ebs'] = click.confirm('Do you want database data directory on external (EBS) storage? [Yes]',
                                             default=defaults['use_ebs'])
    else:
        variables['use_ebs'] = True

    if variables['use_ebs']:
        prompt(variables, 'volume_size', 'Database volume size (GB, 10 or more)', default=defaults['volume_size'])
        prompt(variables, 'volume_type', 'Database volume type (gp2, io1 or standard)',
               default=defaults['volume_type'])
        if variables['volume_type'] == 'io1':
            pio_max = variables['volume_size'] * 30
            prompt(variables, "volume_iops", 'Provisioned I/O operations per second (100 - {0})'.
                   format(pio_max), default=str(pio_max))
        prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
        if ebs_optimized_supported(variables['instance_type']):
            variables['ebs_optimized'] = True
    prompt(variables, "fstype", "Filesystem for the data partition", default=defaults['fstype'])
    prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)",
           default=defaults['fsoptions'])
    prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")

    prompt(variables, 'pgpassword_superuser', "Password for PostgreSQL superuser [random]", show_default=False,
           default=generate_random_password, hide_input=True, confirmation_prompt=True)
    prompt(variables, 'pgpassword_standby', "Password for PostgreSQL user standby [random]", show_default=False,
           default=generate_random_password, hide_input=True, confirmation_prompt=True)
    prompt(variables, 'pgpassword_admin', "Password for PostgreSQL user admin", show_default=True,
           default=defaults['pgpassword_admin'], hide_input=True, confirmation_prompt=True)

    if click.confirm('Do you wish to encrypt these passwords using KMS?', default=False):
        kms_keys = [k for k in list_kms_keys(region) if 'alias/aws/ebs' not in k['aliases']]

        if len(kms_keys) == 0:
            raise click.UsageError('No KMS key is available for encrypting and decrypting. '
                                   'Ensure you have at least 1 key available.')

        options = ['{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys]
        kms_key = choice(prompt='Please select the encryption key', options=options)
        kms_keyid = kms_key.split(':')[0]

        variables['kms_arn'] = [k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid][0]

        for key in [k for k in variables if k.startswith('pgpassword_') or k == 'scalyr_account_key']:
            if variables[key]:
                encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
                variables[key] = 'aws:kms:{}'.format(encrypted)

    set_default_variables(variables)

    check_s3_bucket(variables['wal_s3_bucket'], region)

    return variables
Пример #4
0
def gather_user_variables(variables, region, account_info):
    defaults = set_default_variables(dict())

    if click.confirm('Do you want to set the docker image now? [No]'):
        prompt(variables,
               "docker_image",
               "Docker Image Version",
               default=get_latest_image())

    prompt(variables,
           'wal_s3_bucket',
           'Postgres WAL S3 bucket to use',
           default='{}-{}-spilo-app'.format(get_account_alias(), region))

    prompt(variables,
           'instance_type',
           'EC2 instance type',
           default='t2.medium')

    variables['hosted_zone'] = account_info.Domain or defaults['hosted_zone']
    if (variables['hosted_zone'][-1:] != '.'):
        variables['hosted_zone'] += '.'
    prompt(variables,
           'discovery_domain',
           'ETCD Discovery Domain',
           default='postgres.' + variables['hosted_zone'][:-1])

    variables['add_replica_loadbalancer'] = click.confirm(
        'Do you want a replica ELB?', default=False)

    prompt(variables,
           'elb_access_cidr',
           'Which network should be allowed to access the ELB'
           's? (default=vpc)',
           default=get_vpc_attribute(region=region,
                                     vpc_id=account_info.VpcID,
                                     attribute='cidr_block'))

    odd_sg_name = 'Odd (SSH Bastion Host)'
    odd_sg = get_security_group(region, odd_sg_name)
    if odd_sg and click.confirm(
            'Do you want to allow access to the Spilo nodes from {}?'.format(
                odd_sg_name),
            default=True):
        variables['odd_sg_id'] = odd_sg.group_id

    # Find all Security Groups attached to the zmon worker with 'zmon' in their name
    ec2 = boto3.client('ec2', region)
    filters = [{
        'Name': 'tag-key',
        'Values': ['StackName']
    }, {
        'Name': 'tag-value',
        'Values': ['zmon-worker']
    }]
    zmon_sgs = list()
    for reservation in ec2.describe_instances(Filters=filters).get(
            'Reservations', []):
        for instance in reservation.get('Instances', []):
            zmon_sgs += [
                sg['GroupId'] for sg in instance.get('SecurityGroups', [])
                if 'zmon' in sg['GroupName']
            ]

    if len(zmon_sgs) == 0:
        warning('Could not find zmon security group')
    else:
        click.confirm(
            'Do you want to allow access to the Spilo nodes from zmon?',
            default=True)
        if len(zmon_sgs) > 1:
            prompt(
                variables, 'zmon_sg_id',
                'Which Security Group should we allow access from? {}'.format(
                    zmon_sgs))
        else:
            variables['zmon_sg_id'] = zmon_sgs[0]

    if variables['instance_type'].lower().split('.')[0] in ('c3', 'g2', 'hi1',
                                                            'i2', 'm3', 'r3'):
        variables['use_ebs'] = click.confirm(
            'Do you want database data directory on external (EBS) storage? [Yes]',
            default=defaults['use_ebs'])
    else:
        variables['use_ebs'] = True

    if variables['use_ebs']:
        prompt(variables,
               'volume_size',
               'Database volume size (GB, 10 or more)',
               default=defaults['volume_size'])
        prompt(variables,
               'volume_type',
               'Database volume type (gp2, io1 or standard)',
               default=defaults['volume_type'])
        if variables['volume_type'] == 'io1':
            pio_max = variables['volume_size'] * 30
            prompt(variables,
                   "volume_iops",
                   'Provisioned I/O operations per second (100 - {0})'.format(
                       pio_max),
                   default=str(pio_max))
        prompt(variables,
               "snapshot_id",
               "ID of the snapshot to populate EBS volume from",
               default="")
        if ebs_optimized_supported(variables['instance_type']):
            variables['ebs_optimized'] = True
    prompt(variables,
           "fstype",
           "Filesystem for the data partition",
           default=defaults['fstype'])
    prompt(variables,
           "fsoptions",
           "Filesystem mount options (comma-separated)",
           default=defaults['fsoptions'])
    prompt(variables, "scalyr_account_key",
           "Account key for your scalyr account", "")

    prompt(variables,
           'pgpassword_superuser',
           "Password for PostgreSQL superuser [random]",
           show_default=False,
           default=generate_random_password,
           hide_input=True,
           confirmation_prompt=True)
    prompt(variables,
           'pgpassword_standby',
           "Password for PostgreSQL user standby [random]",
           show_default=False,
           default=generate_random_password,
           hide_input=True,
           confirmation_prompt=True)
    prompt(variables,
           'pgpassword_admin',
           "Password for PostgreSQL user admin",
           show_default=True,
           default=defaults['pgpassword_admin'],
           hide_input=True,
           confirmation_prompt=True)

    if click.confirm('Do you wish to encrypt these passwords using KMS?',
                     default=False):
        kms_keys = [
            k for k in list_kms_keys(region)
            if 'alias/aws/ebs' not in k['aliases']
        ]

        if len(kms_keys) == 0:
            raise click.UsageError(
                'No KMS key is available for encrypting and decrypting. '
                'Ensure you have at least 1 key available.')

        options = [
            '{}: {}'.format(k['KeyId'], k['Description']) for k in kms_keys
        ]
        kms_key = choice(prompt='Please select the encryption key',
                         options=options)
        kms_keyid = kms_key.split(':')[0]

        variables['kms_arn'] = [
            k['Arn'] for k in kms_keys if k['KeyId'] == kms_keyid
        ][0]

        for key in [
                k for k in variables
                if k.startswith('pgpassword_') or k == 'scalyr_account_key'
        ]:
            encrypted = encrypt(region=region,
                                KeyId=kms_keyid,
                                Plaintext=variables[key],
                                b64encode=True)
            variables[key] = 'aws:kms:{}'.format(encrypted)

    set_default_variables(variables)

    check_s3_bucket(variables['wal_s3_bucket'], region)

    return variables
Пример #5
0
def gather_user_variables(variables, region, account_info):
    defaults = set_default_variables(dict())

    if click.confirm("Do you want to set the docker image now? [No]"):
        prompt(variables, "docker_image", "Docker Image Version", default=get_latest_spilo_image())

    prompt(
        variables,
        "wal_s3_bucket",
        "Postgres WAL S3 bucket to use",
        default="{}-{}-spilo-app".format(get_account_alias(), region),
    )

    prompt(variables, "instance_type", "EC2 instance type", default="t2.micro")

    variables["hosted_zone"] = account_info.Domain or defaults["hosted_zone"]
    if variables["hosted_zone"][-1:] != ".":
        variables["hosted_zone"] += "."
    prompt(variables, "discovery_domain", "ETCD Discovery Domain", default="postgres." + variables["hosted_zone"][:-1])

    variables["add_replica_loadbalancer"] = click.confirm("Do you want a replica ELB?", default=False)

    prompt(
        variables,
        "elb_access_cidr",
        "Which network should be allowed to access the ELB" "s? (default=vpc)",
        default=get_vpc_attribute(region=region, vpc_id=account_info.VpcID, attribute="cidr_block"),
    )

    odd_sg_name = "Odd (SSH Bastion Host)"
    odd_sg = get_security_group(region, odd_sg_name)
    if odd_sg and click.confirm(
        "Do you want to allow access to the Spilo nodes from {}?".format(odd_sg_name), default=True
    ):
        variables["odd_sg_id"] = odd_sg.group_id

    # Find all Security Groups attached to the zmon worker with 'zmon' in their name
    ec2 = boto3.client("ec2", region)
    filters = [{"Name": "tag-key", "Values": ["StackName"]}, {"Name": "tag-value", "Values": ["zmon-worker"]}]
    zmon_sgs = list()
    for reservation in ec2.describe_instances(Filters=filters).get("Reservations", []):
        for instance in reservation.get("Instances", []):
            zmon_sgs += [sg["GroupId"] for sg in instance.get("SecurityGroups", []) if "zmon" in sg["GroupName"]]

    if len(zmon_sgs) == 0:
        warning("Could not find zmon security group")
    else:
        click.confirm("Do you want to allow access to the Spilo nodes from zmon?", default=True)
        if len(zmon_sgs) > 1:
            prompt(variables, "zmon_sg_id", "Which Security Group should we allow access from? {}".format(zmon_sgs))
        else:
            variables["zmon_sg_id"] = zmon_sgs[0]

    if variables["instance_type"].lower().split(".")[0] in ("c3", "g2", "hi1", "i2", "m3", "r3"):
        variables["use_ebs"] = click.confirm(
            "Do you want database data directory on external (EBS) storage? [Yes]", default=defaults["use_ebs"]
        )
    else:
        variables["use_ebs"] = True

    if variables["use_ebs"]:
        prompt(variables, "volume_size", "Database volume size (GB, 10 or more)", default=defaults["volume_size"])
        prompt(variables, "volume_type", "Database volume type (gp2, io1 or standard)", default=defaults["volume_type"])
        if variables["volume_type"] == "io1":
            pio_max = variables["volume_size"] * 30
            prompt(
                variables,
                "volume_iops",
                "Provisioned I/O operations per second (100 - {0})".format(pio_max),
                default=str(pio_max),
            )
        prompt(variables, "snapshot_id", "ID of the snapshot to populate EBS volume from", default="")
        if ebs_optimized_supported(variables["instance_type"]):
            variables["ebs_optimized"] = True
    prompt(variables, "fstype", "Filesystem for the data partition", default=defaults["fstype"])
    prompt(variables, "fsoptions", "Filesystem mount options (comma-separated)", default=defaults["fsoptions"])
    prompt(variables, "scalyr_account_key", "Account key for your scalyr account", "")

    prompt(
        variables,
        "pgpassword_superuser",
        "Password for PostgreSQL superuser [random]",
        show_default=False,
        default=generate_random_password,
        hide_input=True,
        confirmation_prompt=True,
    )
    prompt(
        variables,
        "pgpassword_standby",
        "Password for PostgreSQL user standby [random]",
        show_default=False,
        default=generate_random_password,
        hide_input=True,
        confirmation_prompt=True,
    )
    prompt(
        variables,
        "pgpassword_admin",
        "Password for PostgreSQL user admin",
        show_default=True,
        default=defaults["pgpassword_admin"],
        hide_input=True,
        confirmation_prompt=True,
    )

    if click.confirm("Do you wish to encrypt these passwords using KMS?", default=False):
        kms_keys = [k for k in list_kms_keys(region) if "alias/aws/ebs" not in k["aliases"]]

        if len(kms_keys) == 0:
            raise click.UsageError(
                "No KMS key is available for encrypting and decrypting. " "Ensure you have at least 1 key available."
            )

        options = ["{}: {}".format(k["KeyId"], k["Description"]) for k in kms_keys]
        kms_key = choice(prompt="Please select the encryption key", options=options)
        kms_keyid = kms_key.split(":")[0]

        variables["kms_arn"] = [k["Arn"] for k in kms_keys if k["KeyId"] == kms_keyid][0]

        for key in [k for k in variables if k.startswith("pgpassword_")]:
            encrypted = encrypt(region=region, KeyId=kms_keyid, Plaintext=variables[key], b64encode=True)
            variables[key] = "aws:kms:{}".format(encrypted)

    set_default_variables(variables)

    check_s3_bucket(variables["wal_s3_bucket"], region)

    return variables