def add(s3_path, kms_alias, region, variables):
"""
Set/Update environment variables.
:param str s3_path: The full path where the env file is located.
:param str kms_alias: Alias of the KMS key to be used.
:param str region: AWS Region for KMS and S3.
:param str variables: Environment variable name-value pairs to add/update.
"""
# If no variables were given on the command-line, read from STDIN.
if not variables:
parsed_variables = parse_environment_variable_name_value_pairs(None)
else:
parsed_variables = {}
for name_value_pair in variables:
parsed_variables.update(
parse_environment_variable_name_value_pairs(name_value_pair))
environment_storage = EnvironmentStorage(kms_alias=kms_alias,
region=region)
try:
environment_storage.add(s3_path=s3_path, variables=parsed_variables)
except KMSAliasNotFoundError as e:
if e.message.endswith(': None'):
message = \
'Configuration file does not exist. For initialization ' \
'please provide the KMS alias using the ' \
'--kms-alias parameter ({}).'.format(e.message)
else:
message = 'Invalid KMS alias: {}'.format(e.message)
raise click.ClickException(message)
def get(s3_path, region, variable_names):
"""
Get one or many environment variables.
:param str s3_path: The full path where the env file is located.
:param str region: AWS Region for KMS and S3.
:param (str,) variable_names: Which variables to display. If none is
provided, all are displayed.
"""
environment_storage = EnvironmentStorage(kms_alias=None, region=region)
try:
stored_variables = environment_storage.get(s3_path=s3_path)
except ConfigurationFileNotFoundError:
raise click.Abort(
"Access denied or S3 path does not exist: '{}'".format(s3_path))
# Print all configuration if no variables are given
if not variable_names:
variable_names = stored_variables.keys()
for name in variable_names:
try:
# The default log stream is STDERR, so by printing environment
# variables in STDOUT we can easily redirect all logs
# and read only variables.
click.echo('{}={}'.format(quote(name),
quote(stored_variables[name])))
except KeyError:
raise click.Abort(
"No environment variable named '{}'.".format(name))
def test_output(self):
environment = EnvironmentStorage(kms_alias='test', display=mock.Mock())
environment._output('test-1', 'error')
environment._logger.failed.assert_called_once_with('test-1')
with mock.patch('logging.Logger.log') as mocked_log:
environment = EnvironmentStorage(kms_alias='test')
test_cases = (('error', logging.ERROR), ('info', logging.INFO),
('debug', logging.DEBUG))
for level, expected_level in test_cases:
environment._output('test', level)
mocked_log.assert_called_with(msg='test', level=expected_level)
def test_init(self):
environment = EnvironmentStorage(kms_alias=self.kms_alias)
self.assertIsInstance(environment._logger, Logger)
logger_configurator = \
'myev.environment.EnvironmentStorage._configure_logger'
with mock.patch(logger_configurator) as \
mocked:
stream = StringIO()
_ = EnvironmentStorage(kms_alias=self.kms_alias, stream=stream)
mocked.assert_called_once_with(stream=stream)
_ = EnvironmentStorage(kms_alias=self.kms_alias)
mocked.assert_called_with(stream=sys.stderr)
def setUp(self):
self.kms_alias = 'test-kms-alias'
self.prefixed_kms_alias = 'alias/{}'.format(self.kms_alias)
self.kms_key_list = {
'Aliases': [
{
'AliasName': self.prefixed_kms_alias,
'TargetKeyId': '3d78336b-f5c1-4bde-ab0e-1a8a2d3f1c4a'
},
{
'AliasName': 'alias/another-kms-alias',
'TargetKeyId': 'id-2'
},
]
}
self.environment = EnvironmentStorage(kms_alias=self.kms_alias)
# Sample data encrypted and uploaded by the Ruby SDK
self.s3_metadata = {
'x-amz-cek-alg':
'AES/CBC/PKCS5Padding',
'x-amz-iv':
'wyqRlcTfa8qtvYc2AZM1vg==',
'x-amz-key-v2':
'AQEDAHgwrsK9FliCzxfzK2/qJ7ZEc5ZWM8q9WzJgY/ldZTG'
'jigAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAW'
'UDBAEuMBEEDJ2oZHkbpznDeSlu3AIBEIA7C9G+Y5BtdSsd2hns8r3dvQKzZ0R/fV'
'd9gbiHZ7/aIoC0+Hwthq+/grWjWlQvOnfDk9F8sK7dO7rWXeM=',
'x-amz-matdesc':
'{"kms_cmk_id":"3d78336b-f5c1-4bde-ab0e-1a8a2d3f1c4a"}',
'x-amz-unencrypted-content-length':
'45',
'x-amz-wrap-alg':
'kms'
}
self.plaintext_data = '{"A": "1", "B": "abcdefghijklmnopqrstuvxyz"}\n'
self.base64_encrypted_data = \
'Vjg8GFTZds+gVhCTP9H+YgfiVifJDSCljtwde46LSLesamFf1Vfz/4v1YGITDlUM'
self.base64_plaintext_envelope_key = \
'tUQZbnbmv1woZhiW7UulhWe3xfNHERnxRoVcjHk5a7w='
self.s3_bucket = 'myev-tests-123'
def delete(s3_path, region, variables, prompt):
"""
Remove environment variables.
:param str s3_path: The full path where the env file is located.
:param str kms_alias: Alias of the KMS key to be used.
:param str variables: Which env variables to delete.
:param str region: Region of the KMS-key.
:param bool prompt: whether to prompt for confirmation or not before
deleting the variables.
"""
environment_storage = EnvironmentStorage(kms_alias=None, region=region)
if not variables:
raise click.Abort('No variables given for deletion.')
if not prompt or click.confirm(
"Are you sure you want to delete the variables '{}'?".format(
', '.join(variables)),
abort=True):
environment_storage.delete(s3_path=s3_path, variables=variables)
def test_normalize_environment_variables(self):
environment = EnvironmentStorage(kms_alias=self.kms_alias)
with self.assertRaises(
ConfigurationInvalidEnvironmentVariableNameError):
environment._normalize_environment_variables({'-A': '1'})
variables = {'_a0': 'test1', 'v_E_1': 'Test2'}
result = environment._normalize_environment_variables(variables)
self.assertDictEqual(result, {'_A0': 'test1', 'V_E_1': 'Test2'})
def rotate_keys(s3_path, region, new_kms_alias):
# Untested
environment_storage = EnvironmentStorage(kms_alias=None, region=region)
variables = environment_storage.get(s3_path)
environment_storage = EnvironmentStorage(kms_alias=new_kms_alias,
region=region)
environment_storage.add(s3_path, variables)
click.echo("KMS Key for '{}' updated to '{}'.".format(
s3_path, new_kms_alias))
class TestEnvironmentStorage(unittest.TestCase):
def setUp(self):
self.kms_alias = 'test-kms-alias'
self.prefixed_kms_alias = 'alias/{}'.format(self.kms_alias)
self.kms_key_list = {
'Aliases': [
{
'AliasName': self.prefixed_kms_alias,
'TargetKeyId': '3d78336b-f5c1-4bde-ab0e-1a8a2d3f1c4a'
},
{
'AliasName': 'alias/another-kms-alias',
'TargetKeyId': 'id-2'
},
]
}
self.environment = EnvironmentStorage(kms_alias=self.kms_alias)
# Sample data encrypted and uploaded by the Ruby SDK
self.s3_metadata = {
'x-amz-cek-alg':
'AES/CBC/PKCS5Padding',
'x-amz-iv':
'wyqRlcTfa8qtvYc2AZM1vg==',
'x-amz-key-v2':
'AQEDAHgwrsK9FliCzxfzK2/qJ7ZEc5ZWM8q9WzJgY/ldZTG'
'jigAAAH4wfAYJKoZIhvcNAQcGoG8wbQIBADBoBgkqhkiG9w0BBwEwHgYJYIZIAW'
'UDBAEuMBEEDJ2oZHkbpznDeSlu3AIBEIA7C9G+Y5BtdSsd2hns8r3dvQKzZ0R/fV'
'd9gbiHZ7/aIoC0+Hwthq+/grWjWlQvOnfDk9F8sK7dO7rWXeM=',
'x-amz-matdesc':
'{"kms_cmk_id":"3d78336b-f5c1-4bde-ab0e-1a8a2d3f1c4a"}',
'x-amz-unencrypted-content-length':
'45',
'x-amz-wrap-alg':
'kms'
}
self.plaintext_data = '{"A": "1", "B": "abcdefghijklmnopqrstuvxyz"}\n'
self.base64_encrypted_data = \
'Vjg8GFTZds+gVhCTP9H+YgfiVifJDSCljtwde46LSLesamFf1Vfz/4v1YGITDlUM'
self.base64_plaintext_envelope_key = \
'tUQZbnbmv1woZhiW7UulhWe3xfNHERnxRoVcjHk5a7w='
self.s3_bucket = 'myev-tests-123'
def _configure_s3(self, create_bucket=True):
import boto3
s3 = boto3.client('s3')
if create_bucket:
s3.create_bucket(Bucket=self.s3_bucket)
return s3
def _get_s3_path(self, key):
return 's3://{}/{}'.format(self.s3_bucket, key)
def test_path_construction(self):
self.assertEqual(self._get_s3_path('dummy'),
's3://myev-tests-123/dummy')
def test_init(self):
environment = EnvironmentStorage(kms_alias=self.kms_alias)
self.assertIsInstance(environment._logger, Logger)
logger_configurator = \
'myev.environment.EnvironmentStorage._configure_logger'
with mock.patch(logger_configurator) as \
mocked:
stream = StringIO()
_ = EnvironmentStorage(kms_alias=self.kms_alias, stream=stream)
mocked.assert_called_once_with(stream=stream)
_ = EnvironmentStorage(kms_alias=self.kms_alias)
mocked.assert_called_with(stream=sys.stderr)
def test_output(self):
environment = EnvironmentStorage(kms_alias='test', display=mock.Mock())
environment._output('test-1', 'error')
environment._logger.failed.assert_called_once_with('test-1')
with mock.patch('logging.Logger.log') as mocked_log:
environment = EnvironmentStorage(kms_alias='test')
test_cases = (('error', logging.ERROR), ('info', logging.INFO),
('debug', logging.DEBUG))
for level, expected_level in test_cases:
environment._output('test', level)
mocked_log.assert_called_with(msg='test', level=expected_level)
@mock_s3
@mock_kms
def test_normalize_environment_variables(self):
environment = EnvironmentStorage(kms_alias=self.kms_alias)
with self.assertRaises(
ConfigurationInvalidEnvironmentVariableNameError):
environment._normalize_environment_variables({'-A': '1'})
variables = {'_a0': 'test1', 'v_E_1': 'Test2'}
result = environment._normalize_environment_variables(variables)
self.assertDictEqual(result, {'_A0': 'test1', 'V_E_1': 'Test2'})
@mock_s3
def test_get_bucket_does_not_exist(self):
s3 = self._configure_s3(create_bucket=False)
with self.assertRaises(ConfigurationFileNotFoundError):
self.environment.get(self._get_s3_path('dummy'))
@mock_s3
def test_get_key_does_not_exist(self):
self._configure_s3()
with self.assertRaises(ConfigurationFileNotFoundError):
self.environment.get(self._get_s3_path('dummy'))
@mock_s3
def test_get(self):
s3 = self._configure_s3()
self.environment._encryption_backend.get_object = mock.MagicMock(
return_value={
'Body': self.plaintext_data,
'Metadata': {
'm1': '1',
'm2': '2'
}
})
result = self.environment.get(self._get_s3_path('dummy'))
self.assertDictEqual(result, json.loads(self.plaintext_data))
@mock_s3
@mock_kms
def test_get_functional(self):
s3 = self._configure_s3(create_bucket=True)
s3_key = 'dummy'
s3.put_object(Body=base64.b64decode(self.base64_encrypted_data),
Bucket=self.s3_bucket,
Key=s3_key,
Metadata=self.s3_metadata,
ServerSideEncryption='AES256')
self.environment._encryption_backend._key_provider.decrypt = \
mock.MagicMock(
return_value=base64.b64decode(
self.base64_plaintext_envelope_key
)
)
self.assertEqual(self.environment.get(self._get_s3_path(s3_key)),
json.loads(self.plaintext_data))
@mock_s3
@mock_kms
def test_delete(self):
s3_key = 'dummy'
mock_s3_put_object = mock.MagicMock(return_value={
'Body': 'test',
'Metadata': {
't1': '1',
't2': '2'
}
})
self.environment._encryption_backend.put_object = mock_s3_put_object
with mock.patch.object(EnvironmentStorage, 'get') as mock_get:
mock_get.return_value = {'A_VAR': '1', 'B_VAR': '2', 'C_VAR': '3'}
s3_path = self._get_s3_path(s3_key)
remove_variables = ('B_VAR', 'NOT_FOUND_VAR')
result = self.environment.delete(s3_path, remove_variables)
expected_environment = {
key: value
for key, value in mock_get.return_value.items()
if key != 'B_VAR'
}
self.assertDictEqual(expected_environment, result)
mock_s3_put_object.assert_called_once_with(
bucket=self.s3_bucket,
key=s3_key,
body=json.dumps(expected_environment).replace(' ', ''))
@mock_s3
@mock_kms
def test_add_create(self):
s3 = self._configure_s3()
data = {'A': '1', 'B': '2', 'c': '3'}
s3_key = 'dummy'
mock_put_object = mock.MagicMock()
self.environment._encryption_backend.put_object = mock_put_object
result = self.environment.add(self._get_s3_path(s3_key), data)
mock_put_object.assert_called_once_with(
bucket=self.s3_bucket,
key=s3_key,
body='{"A":"1","C":"3","B":"2"}')
self.assertDictEqual(result, {"A": "1", "B": "2", "C": "3"})
@mock_s3
@mock_kms
@mock.patch.object(environment, 'to_json', patched_to_json)
def test_add_create_functional(self):
s3 = self._configure_s3()
s3_key = 'dummy'
key_provider = self.environment._encryption_backend._key_provider
key_provider._client.generate_data_key = mock.MagicMock(
return_value={
'Plaintext':
base64.b64decode(self.base64_plaintext_envelope_key),
'CiphertextBlob':
base64.b64decode(self.s3_metadata['x-amz-key-v2'])
})
key_provider._client.list_aliases = mock.MagicMock(
return_value=self.kms_key_list)
iv = base64.b64decode(self.s3_metadata['x-amz-iv'])
crypto_random = StringIO(iv)
with mock.patch.object(Random, 'new', return_value=crypto_random):
self.assertDictEqual(
self.environment.add(self._get_s3_path(s3_key),
json.loads(self.plaintext_data)),
json.loads(self.plaintext_data))
s3_object = s3.get_object(Bucket=self.s3_bucket, Key=s3_key)
self.assertDictEqual(s3_object['Metadata'], self.s3_metadata)
self.assertIsInstance(s3_object, dict)
self.assertEqual(s3_object['Body'].read(),
base64.b64decode(self.base64_encrypted_data))
@mock_s3
@mock_kms
def test_add_update(self):
s3 = self._configure_s3()
# Cases:
# - Existing variable, update value (A)
# - Existing variable, unaffected value (B)
# - New variable, normalized name (C)
data = {'A': '3', 'B': '2', 'c': '4'}
s3_key = 'dummy'
mock_get_object = mock.MagicMock(
return_value={'Body': '{"A": "1", "B": "2"}'})
self.environment._encryption_backend.get_object = mock_get_object
mock_put_object = mock.MagicMock()
self.environment._encryption_backend.put_object = mock_put_object
result = self.environment.add(self._get_s3_path(s3_key), data)
mock_get_object.assert_called_once_with(bucket=self.s3_bucket,
key=s3_key)
mock_put_object.assert_called_once_with(
bucket=self.s3_bucket,
key=s3_key,
body='{"A":"3","C":"4","B":"2"}')
self.assertDictEqual(result, {"A": "3", "B": "2", "C": "4"})