class AzureRMKeyVaultKey(AzureRMModuleBase):
''' Module that creates or deletes keys in Azure KeyVault '''
def __init__(self):
self.module_arg_spec = dict(
key_name=dict(type='str', required=True),
keyvault_uri=dict(type='str', required=True),
pem_file=dict(type='str'),
pem_password=dict(type='str'),
byok_file=dict(type='str'),
state=dict(type='str', default='present', choices=['present', 'absent'])
)
self.results = dict(
changed=False,
state=dict()
)
self.key_name = None
self.keyvault_uri = None
self.pem_file = None
self.pem_password = None
self.state = None
self.client = None
self.tags = None
required_if = [
('pem_password', 'present', ['pem_file'])
]
super(AzureRMKeyVaultKey, self).__init__(self.module_arg_spec,
supports_check_mode=True,
required_if=required_if,
supports_tags=True)
def exec_module(self, **kwargs):
for key in list(self.module_arg_spec.keys()) + ['tags']:
setattr(self, key, kwargs[key])
# Create KeyVaultClient
def auth_callback(server, resource, scope):
if self.credentials['client_id'] is None or self.credentials['secret'] is None:
self.fail('Please specify client_id, secret and tenant to access azure Key Vault.')
tenant = self.credentials.get('tenant')
if not self.credentials['tenant']:
tenant = "common"
authcredential = ServicePrincipalCredentials(
client_id=self.credentials['client_id'],
secret=self.credentials['secret'],
tenant=tenant,
resource="https://vault.azure.net")
token = authcredential.token
return token['token_type'], token['access_token']
self.client = KeyVaultClient(KeyVaultAuthentication(auth_callback))
results = dict()
changed = False
try:
results['key_id'] = self.get_key(self.key_name)
# Key exists and will be deleted
if self.state == 'absent':
changed = True
except KeyVaultErrorException:
# Key doesn't exist
if self.state == 'present':
changed = True
self.results['changed'] = changed
self.results['state'] = results
if not self.check_mode:
# Create key
if self.state == 'present' and changed:
results['key_id'] = self.create_key(self.key_name, self.tags)
self.results['state'] = results
self.results['state']['status'] = 'Created'
# Delete key
elif self.state == 'absent' and changed:
results['key_id'] = self.delete_key(self.key_name)
self.results['state'] = results
self.results['state']['status'] = 'Deleted'
else:
if self.state == 'present' and changed:
self.results['state']['status'] = 'Created'
elif self.state == 'absent' and changed:
self.results['state']['status'] = 'Deleted'
return self.results
def get_key(self, name, version=''):
''' Gets an existing key '''
key_bundle = self.client.get_key(self.keyvault_uri, name, version)
if key_bundle:
key_id = KeyVaultId.parse_key_id(key_bundle.key.kid)
return key_id.id
def create_key(self, name, tags, kty='RSA'):
''' Creates a key '''
key_bundle = self.client.create_key(vault_base_url=self.keyvault_uri, key_name=name, kty=kty, tags=tags)
key_id = KeyVaultId.parse_key_id(key_bundle.key.kid)
return key_id.id
def delete_key(self, name):
''' Deletes a key '''
deleted_key = self.client.delete_key(self.keyvault_uri, name)
key_id = KeyVaultId.parse_key_id(deleted_key.key.kid)
return key_id.id
def import_key(self, key_name, destination=None, key_ops=None, disabled=False, expires=None,
not_before=None, tags=None, pem_file=None, pem_password=None, byok_file=None):
""" Import a private key. Supports importing base64 encoded private keys from PEM files.
Supports importing BYOK keys into HSM for premium KeyVaults. """
def _to_bytes(hex_string):
# zero pads and decodes a hex string
if len(hex_string) % 2:
hex_string = '{0}'.format(hex_string)
return codecs.decode(hex_string, 'hex_codec')
def _set_rsa_parameters(dest, src):
# map OpenSSL parameter names to JsonWebKey property names
conversion_dict = {
'modulus': 'n',
'publicExponent': 'e',
'privateExponent': 'd',
'prime1': 'p',
'prime2': 'q',
'exponent1': 'dp',
'exponent2': 'dq',
'coefficient': 'qi'
}
# regex: looks for matches that fit the following patterns:
# integerPattern: 65537 (0x10001)
# hexPattern:
# 00:a0:91:4d:00:23:4a:c6:83:b2:1b:4c:15:d5:be:
# d8:87:bd:c9:59:c2:e5:7a:f5:4a:e7:34:e8:f0:07:
# The desired match should always be the first component of the match
regex = re.compile(r'([^:\s]*(:[^\:)]+\))|([^:\s]*(:\s*[0-9A-Fa-f]{2})+))')
# regex2: extracts the hex string from a format like: 65537 (0x10001)
regex2 = re.compile(r'(?<=\(0x{1})([0-9A-Fa-f]*)(?=\))')
key_params = crypto.dump_privatekey(crypto.FILETYPE_TEXT, src).decode('utf-8')
for match in regex.findall(key_params):
comps = match[0].split(':', 1)
name = conversion_dict.get(comps[0], None)
if name:
value = comps[1].replace(' ', '').replace('\n', '').replace(':', '')
try:
value = _to_bytes(value)
except Exception: # pylint:disable=broad-except
# if decoding fails it is because of an integer pattern. Extract the hex
# string and retry
value = _to_bytes(regex2.findall(value)[0])
setattr(dest, name, value)
key_attrs = KeyAttributes(not disabled, not_before, expires)
key_obj = JsonWebKey(key_ops=key_ops)
if pem_file:
key_obj.kty = 'RSA'
with open(pem_file, 'r') as f:
pem_data = f.read()
# load private key and prompt for password if encrypted
try:
pem_password = str(pem_password).encode() if pem_password else None
# despite documentation saying password should be a string, it needs to actually
# be UTF-8 encoded bytes
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, pem_data, pem_password)
except crypto.Error:
pass # wrong password
except TypeError:
pass # no pass provided
_set_rsa_parameters(key_obj, pkey)
elif byok_file:
with open(byok_file, 'rb') as f:
byok_data = f.read()
key_obj.kty = 'RSA-HSM'
key_obj.t = byok_data
return self.client.import_key(
self.keyvault_uri, key_name, key_obj, destination == 'hsm', key_attrs, tags)
from azure.keyvault import KeyVaultClient, KeyVaultAuthentication
from azure.common.credentials import ServicePrincipalCredentials
credentials = ServicePrincipalCredentials(
client_id='YourClientAdd',
secret='YourSecret',
tenant='domain.onmicrosoft.com',
)
VAULT_URL = 'https://YourKeyvaultNAME.vault.azure.net'
SECRET_ID = 'mysecret'
client = KeyVaultClient(credentials)
# VAULT_URL must be in the format 'https://<vaultname>.vault.azure.net'
# SECRET_VERSION is required, and can be obtained with the KeyVaultClient.get_secret_versions(self, vault_url, secret_id) API
key_bundle = client.get_key(VAULT_URL, 'key',
'e371b887e1f54708a947cc8570272543')
key = key_bundle.key
secret_bundle = client.get_secret(VAULT_URL, SECRET_ID,
'90cdeb79bc494b77920ed2bbdba8139e')
#Create a Secret
my_secret = client.set_secret(VAULT_URL, "secretfrompython2", "pythonvalue2")
#Create a Key
key2 = client.create_key(VAULT_URL, "key2", "RSA")
versions = client.get_key_versions(VAULT_URL, SECRET_ID)
#Delete a Key delete_key(vault_base_url, key_name, custom_headers=None, raw=False, **operation_config)
#client.delete_key(VAULT_URL,"key2")
print(my_secret)
print(secret_bundle)
print(key)
print(key2)
# cache the response
_token_cache[token_cache_key] = token
# return a new AccessToken with the scheme, AAD access_token, and POP key
return AccessToken(scheme, token['access_token'],
token.get('pop_key', None))
if __name__ == "__main__":
# create the key vault client with our PoP enabled AAD callback
client = KeyVaultClient(KeyVaultAuthentication(_authenticate))
# create a key to perform server side crypto operations
key_name = _random_name('key')
client.create_key(_get_vault_url(), key_name, 'RSA')
print('Created key vault key: ' + key_name)
# create a random AES key to wrap
to_wrap = os.urandom(16)
print('\noriginal value:')
print(to_wrap)
# wrap the key
wrapped = client.wrap_key(vault_base_url=_get_vault_url(),
key_name=key_name,
key_version='',
algorithm='RSA-OAEP',
value=to_wrap)
class KV_Repl(object):
_repl_break_commands = set(('back', 'b'))
_repl_quit_commands = set(('quit', 'q'))
def __init__(self, config):
self._auth = KeyVaultAuth(config, CLIENT_ID)
self._config = config
self._mgmt_client = KeyVaultManagementClient(
self._auth.get_arm_creds(), config.subscription_id)
self._data_client = KeyVaultClient(self._auth.get_keyvault_creds())
self._selected_vault = None
self._current_index = None
def start(self):
try:
self._vault_index_loop()
except SystemExit:
print('\nuser exited\n')
def _continue_repl(self, display_action, break_commands=()):
display_action()
self._selection = input('> ').lower()
if self._selection in break_commands:
return None
elif self._selection in KV_Repl._repl_quit_commands:
sys.exit()
try:
self._selection = int(self._selection)
except ValueError:
pass
return self._selection
def _display_vault_index(self):
print('\nAvailable Vaults:\n')
self._current_index = self._get_vault_list()
for idx, vault in enumerate(self._current_index):
print('%d. %s' % (idx, vault.name))
print('\n#:select | (a)dd | (d)elete | (q)uit')
def _vault_index_loop(self):
while self._continue_repl(self._display_vault_index) is not None:
vaults = self._current_index
if isinstance(self._selection, int):
i = self._selection
if i >= 0 and i < len(vaults):
self._selected_vault = self._mgmt_client.vaults.get(
self._config.resource_group, vaults[i].name)
self._vault_detail_loop()
else:
print('invalid vault index')
elif self._selection == 'a' or self._selection == 'add':
self._add_vault()
else:
print('invalid input')
def _add_vault(self):
name = input('\nenter vault name:')
all_perms = Permissions()
all_perms.keys = [KeyPermissions.all]
all_perms.secrets = [SecretPermissions.all]
all_perms.certificates = [CertificatePermissions.all]
user_policy = AccessPolicyEntry(self._config.tenant_id,
self._config.user_oid, all_perms)
app_policy = AccessPolicyEntry(CLIENT_TENANT_ID, CLIENT_OID, all_perms)
access_policies = [user_policy, app_policy]
properties = VaultProperties(self._config.tenant_id,
Sku(name='standard'), access_policies)
properties.enabled_for_deployment = True
properties.enabled_for_disk_encryption = True
properties.enabled_for_template_deployment = True
vault = VaultCreateOrUpdateParameters(self._config.location,
properties)
self._mgmt_client.vaults.create_or_update(self._config.resource_group,
name, vault)
print('vault %s created\n' % name)
def _display_selected_vault_detail(self):
print('\nName:\t%s' % self._selected_vault.name)
print('Uri:\t%s' % self._selected_vault.properties.vault_uri)
print('Id:\t%s' % self._selected_vault.id)
print(
'\n(s)ecrets | (k)eys | (c)ertificates | (e)ncrypt | (d)ecrypt | (b)ack | (q)uit\n'
)
def _vault_detail_loop(self):
while self._continue_repl(
self._display_selected_vault_detail,
break_commands=KV_Repl._repl_break_commands) is not None:
if self._selection == 's' or self._selection == 'secrets':
self._secret_index_loop()
elif self._selection == 'k' or self._selection == 'keys':
self._key_index_loop()
elif self._selection == 'c' or self._selection == 'certificates':
print('\nnot yet implemented\n')
elif self._selection == 'e' or self._selection == 'encrypt':
self._encrypt_file()
else:
print('invalid input')
def _encrypt_file(self):
while True:
inpath = input('input file: ')
if os.path.isfile(inpath):
break
else:
print('error: file not found')
while True:
outpath = input('output file: ')
@staticmethod
def _prompt_for_file_path(prompt, verify_exists):
inpath = input(prompt)
def _display_secret_index(self):
self._current_index = []
secret_iter = self._data_client.get_secrets(
self._selected_vault.properties.vault_uri)
if secret_iter is not None:
try:
self._current_index = [secret for secret in secret_iter]
except TypeError:
pass
print('\n%s Secrets:\n' % self._selected_vault.name)
for idx, s in enumerate(self._current_index):
print('%d. %s' % (idx, KV_Repl._get_name_from_url(s.id)))
print('\n#:show secret value (a)dd (d)elete (b)ack (q)uit\n')
def _secret_index_loop(self):
while self._continue_repl(
self._display_secret_index,
break_commands=KV_Repl._repl_break_commands) is not None:
secrets = self._current_index
if isinstance(self._selection, int):
i = self._selection
if i >= 0 and i < len(secrets):
print('\n%s = %s\n' %
(KV_Repl._get_secret_name_from_url(secrets[i].id),
self._data_client.get_secret(secrets[i].id).value))
else:
print('invalid secret index')
elif self._selection == 'a' or self._selection == 'add':
self._add_secret()
elif self._selection == 'd' or self._selection == 'delete':
print('\nnot yet implemented\n')
def _add_secret(self):
secret_name = input('\nSecret Name: ')
secret_value = input('Secret Value: ')
self._data_client.set_secret(self._selected_vault.properties.vault_uri,
secret_name, secret_value)
print('\nSecret %s added to vault %s' %
(secret_name, self._selected_vault.name))
def _display_key_index(self):
self._current_index = []
key_iter = self._data_client.get_keys(
self._selected_vault.properties.vault_uri)
if key_iter is not None:
try:
self._current_index = [secret for secret in key_iter]
except TypeError:
print('warning: caught TypeError')
pass
print('\n%s Keys:\n' % self._selected_vault.name)
for idx, k in enumerate(self._current_index):
print('%d. %s' % (idx, KV_Repl._get_name_from_url(k.kid)))
print('\n#:get key | (a)dd | (i)mport | (d)elete | (b)ack | (q)uit\n')
def _key_index_loop(self):
while self._continue_repl(
self._display_key_index,
break_commands=KV_Repl._repl_break_commands) is not None:
keys = self._current_index
if isinstance(self._selection, int):
i = self._selection
if i >= 0 and i < len(keys):
print('\n%s = %s\n' %
(KV_Repl._get_secret_name_from_url(keys[i].id),
self._data_client.get_secret(keys[i].id).value))
else:
print('invalid key index')
elif self._selection == 'a' or self._selection == 'add':
self._add_key()
elif self._selection == 'd' or self._selection == 'delete':
print('\nnot yet implemented\n')
def _add_key(self):
key_name = input('\nKey Name: ')
self._data_client.create_key(self._selected_vault.properties.vault_uri,
key_name,
kty=JsonWebKeyType.rsa.value)
print('\nSecret %s added to vault %s' %
(key_name, self._selected_vault.name))
@staticmethod
def _get_name_from_url(url):
split = url.split('/')
return split[len(split) - 1]
def _get_vault_list(self):
vault_list = [vault for vault in self._mgmt_client.vaults.list()]
return vault_list
class MgmtHDInsightTest(AzureMgmtTestCase):
def setUp(self):
super(MgmtHDInsightTest, self).setUp()
# create client
self.hdinsight_client = self.create_mgmt_client(HDInsightManagementClient)
self.msi_client = self.create_mgmt_client(ManagedServiceIdentityClient)
self.vault_mgmt_client = self.create_mgmt_client(KeyVaultManagementClient)
if self.is_live:
self.vault_client = self.create_basic_client(KeyVaultClient)
else:
def _auth_callback(server, resource, scope):
return AccessToken('Bearer', 'fake-token')
self.vault_client = KeyVaultClient(KeyVaultAuthentication(authorization_callback=_auth_callback))
# sensitive test configs
self.tenant_id = self.settings.TENANT_ID
self.adls_account_name = self.settings.HDI_ADLS_ACCOUNT_NAME
self.adls_client_id = self.settings.HDI_ADLS_CLIENT_ID
# Non-sensitive test configs
self.cluster_username = '******'
self.cluster_password = '******'
self.ssh_username = '******'
self.ssh_password = '******'
self.adls_home_mountpoint = '/clusters/hdi'
self.cert_password = '******'
self.cert_content = 'MIIJ8gIBAzCCCa4GCSqGSIb3DQEHAaCCCZ8EggmbMIIJlzCCBgAGCSqGSIb3DQEHAaCCBfEEggXtMIIF6TCCBeUGCyqGSIb3DQEMCgECoIIE9jCCBPIwHAYKKoZIhvcNAQwBAzAOBAiTJstpWcGFZAICB9AEggTQZvg9qVE2ptb3hdH9hnDf5pwIeGghe9assBeEKj/W1JMUjsdEu7qzXH9/3Ro6C1HF6MvSqbav7MD8je9AMb0jl7T3ZmXPgGtrbUsSBTPruVv0hTXPRTxQmcfwae5vEkD03b/4W22sXMMYZB7wOTQMl1d5+0wt267qdF+G1XWtRI2jnxetK8/oyMQxn5Cwuz/VAmn1tXwRAN9gIiZDA8MwvpYha/iFVWKu/vnHg8HT47ry+27/wh8ifM9ea7JWhKh2tZoPIMou9/P/CgkkMv9KVHlmiMldA3Phxsrqjbh/wbd8RWBOtSM7rryMVnc1MYonZraDJMGOLGAhvEcXNBKLwRdmrDDYvpOYlDYKlmNvDXYDm410XCOia2aKP0WoP4qLsExtUwW8Qk2r2QRy5O4B5p2EbPZBlPlMMO4S1NkASjJCghuTCUgvk3uwe2L/mMf0IajAf+R0/VW/fXHbhJkFKobi5JlIqWaHsSC7hMidWj6771Yi8UEXOMshWERs2UHH05aIO3c50lnyypHyhA3BohKUXzNhHA0o7ImQRjmjjTJFBLMNiIZSW0aPtLN1+92pT0a6iS7P1PC0DqEnOjkcs/VOUI3qTt0X78b6wnDO+ATo39B13njGD0mtrVvtTeHclBouoFzpKHkS86GSGmDoHQH6EHhpGF/7wPVfAjAiSrNQb/QLjPHWo+BeiOml4Xrti0K6rWb0AXhY8AmtIlEUC9GscDSdT55v9j9tWONzOXECCgZBYDzNlagMLkCDIFHZwbEGPn3pOc7BTOmQf1GQjfvunLiLWWfe3of9pR+NCDyi1VJUNvjoE+/YnVoBBUMBBO6/4t2SL92iouEF4fyqkQFDb0FOPW4Kfh7H5W+sDZIN9NfqNzniK6HFcpS+jnGm9x9rx81DmMcwtiYZTfYDSivtNxOYrmRFXx574stBzvG0En11uc6E4QhWnkCSsBnnOMjRGDyv95BFVMZC0gIS0rWoKYxjdblpmo9w/yfDtAmQuCs3bdqcJ4mMYt0ueUUZImPRQRJOSrVyoq+brLw657EqM1SahtBmzTG7+HTl1Qi/xZ1xmz6paQDSFVPRcb5QSIp4v08j/Lmj0x4R9jQ4cAmZ3CfPKXBKuIRu2AI2EuqGOoAxvQQEpSjSKUs3fbQxjptUhK7o5FcXAfAxHLzdx2/9L1Iqbo/3HDkbmuix24NRXESG0e/kVr7VAGhoALI7L+eKAdn4AkgmBt55FXZ+uHY9bSKZUoz4Oed2bz2A+9sQBcXG06fLqQEwGVPhATEbYyRduuY6AdTRAmOKmadT5BTTD7+dnFlIt+u7ZpbXm6S6LcSqGqHVacig27SwDt0VznQsjMRDVCiHaWKg4W78xbP7YVvNTB/cBCHmhh5ZXfO/TucizXsQPJlwEMr2CbqByqldXi0i1GUrbg4aLUGZtxgUYex7dHlx6GUejOGRh7fLYCNBo43pjCFvbhFwb0/dWya0crJjpGiY3DNtl1YosJBmvso/Rli4QqVeN7tb45ZsGWTEUg1MDeoGRDqal7GDsvBnH574T5Sz3nSLAoNXR7k0rYaWhezJNobo/SDkuSXskVjNJpv+vjEyW2UGYNhaeK/UHKBP8IrmSAEIZejQj6OEzSPM6TNLW5qJb6LK9agxgdswEwYJKoZIhvcNAQkVMQYEBAEAAAAwXQYJKwYBBAGCNxEBMVAeTgBNAGkAYwByAG8AcwBvAGYAdAAgAFMAdAByAG8AbgBnACAAQwByAHkAcAB0AG8AZwByAGEAcABoAGkAYwAgAFAAcgBvAHYAaQBkAGUAcjBlBgkqhkiG9w0BCRQxWB5WAFAAdgBrAFQAbQBwADoAMAAyAGYAZQA0AGUAOAAzAC0AMgAzADEANgAtADQAMQA3AGMALQA5ADQANQBjAC0AZgA1ADUAMABhADUAZAAwAGIAMAAzAGEwggOPBgkqhkiG9w0BBwagggOAMIIDfAIBADCCA3UGCSqGSIb3DQEHATAcBgoqhkiG9w0BDAEDMA4ECAR1hzovrDkgAgIH0ICCA0gq6boOLRoE5PHFfVIXYtzqg1u2vPMm5mgBUvrh3u+VZ/1FMGTj8od9+Yu91cnumVSgfRiGq7lz+K6by5JsBqP68ksLA2d/PqtTdofCoZ7SgVIo+qqzA64HIQFkElNpo/BJMX/JGpc5OlFq7mdHe6xL2Pfx/3z/pNSV+D+WaAwaDnbLqI7MU6ED3j5l63mExk/8H/VVbiPdqMTzbhIp65oHTGanw86w7RlywqeNb3DkPVMB78Jhcg8vf2AxB8hKf6QiO2uJc/4WKkyLoLmNoD/zhaoUuAbC6hrNVAa9VRWNRfwKZrzwIMSLlKYwWmVcD/QgC8gwxuF+wV3UHwDNAdEe8TEsOhE99/ZiUiogxMdkopZwwtZMszzBB/x5mHCGySauDMVPwoYT6QXewJhGrUap0jwB/Vzy5FaWHi/m8964zWpwC6xfkT2hkDb+rfouWutpiAgMne5tD9YvqxTUmZFIlgwpLrVdPcKQS+b/uIXPTv8uW177XsCOmGGu728ld8H1Ifb2nPveK09Y0AA+ARFpOX0p0ZuxMQqk6NnlA+eESJVm5cLfKszorRcrNPINXaEOGl2okuijm8va30FH9GIYWRt+Be6s8qG672aTO/2eHaTHeR/qQ9aEt0ASDXGcugYS14Jnu2wbauyvotZ6eAvgl5tM2leMpgSLaQqYzPTV2uiD6zDUqxwjm2P8EZQihEQqLUV1jvQuQB4Ui7MryDQ+QiDBD2m5c+9SDPafcR7qgRe/cP4hj5BqzHTcNQAD5BLXze7Yx+TMdf+Qe/R1uBYm8bLjUv9hwUmtMeZP4RU6RPJrN2aRf7BUdgS0j/8YAhxViPucRENuhEfS4sotHf1CJZ7xJz0ZE9cpVY6JLl1tbmuf1Eh50cicZ1SHQhqSP0ggLHV6DNcJz+kyekEe9qggGDi6mreYz/kJnnumsDy5cToIHy9jJhtXzj+/ZNGkdpq9HWMiwAT/VR1WPpzjn06m7Z87PiLaiC3simQtjnl0gVF11Ev4rbIhYjFBL0nKfNpzaWlMaOVF+EumROk3EbZVpx1K6Yh0zWh/NocWSUrtSoHVklzwPCNRvnE1Ehyw5t9YbEBsTIDWRYyqbVYoFVfOUhq5p4TXrqEwOzAfMAcGBSsOAwIaBBSx7sJo66zYk4VOsgD9V/co1SikFAQUUvU/kE4wTRnPRnaWXnno+FCb56kCAgfQ'
self.workspace_id = '1d364e89-bb71-4503-aa3d-a23535aea7bd'
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-humboldt')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster_with_premium_tier(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-premium')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.tier=Tier.premium
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_with_empty_extended_parameters(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-cluster')
create_params = ClusterCreateParametersExtended()
# try to create cluster and ensure it throws
try:
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.assertTrue(False, 'should not have made it here')
except Exception:
pass
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster_with_custom_vm_sizes(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-customvmsizes')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
headnode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'headnode')
headnode.hardware_profile = HardwareProfile(vm_size="ExtraLarge")
zookeepernode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'zookeepernode')
zookeepernode.hardware_profile = HardwareProfile(vm_size="Medium")
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_linux_spark_cluster_with_component_version(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-sparkcomponentversions')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Spark'
create_params.properties.cluster_definition.Component_version = {'Spark' : '2.2'}
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_kafka_cluster_with_managed_disks(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-kafka')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Kafka'
workernode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
workernode.data_disks_groups = [
DataDisksGroups(
disks_per_node=8
)
]
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
@KeyVaultPreparer(name_prefix='hdipy', location=LOCATION, enable_soft_delete=True)
def test_create_kafka_cluster_with_disk_encryption(self, resource_group, location, storage_account, storage_account_key, vault):
# create managed identities for Azure resources.
msi_name = self.get_resource_name('hdipyuai')
msi = self.msi_client.user_assigned_identities.create_or_update(resource_group.name, msi_name, location)
# add managed identity to vault
required_permissions = Permissions(keys=[KeyPermissions.get, KeyPermissions.wrap_key, KeyPermissions.unwrap_key],
secrets=[SecretPermissions.get, SecretPermissions.set,SecretPermissions.delete])
vault.properties.access_policies.append(
AccessPolicyEntry(tenant_id=self.tenant_id,
object_id=msi.principal_id,
permissions=required_permissions)
)
update_params = VaultCreateOrUpdateParameters(location=location,
properties=vault.properties)
vault = self.vault_mgmt_client.vaults.create_or_update(resource_group.name, vault.name, update_params).result()
self.assertIsNotNone(vault)
# create key
vault_uri = vault.properties.vault_uri
key_name = self.get_resource_name('hdipykey1')
created_bundle = self.vault_client.create_key(vault_uri, key_name, 'RSA')
vault_key = KeyVaultId.parse_key_id(created_bundle.key.kid)
# create HDInsight cluster with Kafka disk encryption
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-kafka-byok')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Kafka'
workernode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
workernode.data_disks_groups = [
DataDisksGroups(
disks_per_node=8
)
]
create_params.identity = ClusterIdentity(
type=ResourceIdentityType.user_assigned,
user_assigned_identities={msi.id: ClusterIdentityUserAssignedIdentitiesValue()}
)
create_params.properties.disk_encryption_properties = DiskEncryptionProperties(
vault_uri=vault_key.vault,
key_name=vault_key.name,
key_version=vault_key.version,
msi_resource_id=msi.id
)
cluster = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
# check disk encryption properties
self.assertIsNotNone(cluster.properties.disk_encryption_properties)
self.assertEqual(create_params.properties.disk_encryption_properties.vault_uri, cluster.properties.disk_encryption_properties.vault_uri)
self.assertEqual(create_params.properties.disk_encryption_properties.key_name, cluster.properties.disk_encryption_properties.key_name)
self.assertEqual(create_params.properties.disk_encryption_properties.msi_resource_id.lower(), cluster.properties.disk_encryption_properties.msi_resource_id.lower())
# create a new key
new_key_name = self.get_resource_name('hdipykey2')
created_bundle = self.vault_client.create_key(vault_uri, new_key_name, 'RSA')
new_vault_key = KeyVaultId.parse_key_id(created_bundle.key.kid)
rotate_params = ClusterDiskEncryptionParameters(
vault_uri=new_vault_key.vault,
key_name=new_vault_key.name,
key_version=new_vault_key.version
)
# rotate cluster key
self.hdinsight_client.clusters.rotate_disk_encryption_key(rg_name, cluster_name, rotate_params).wait()
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
# check disk encryption properties
self.assertIsNotNone(cluster.properties.disk_encryption_properties)
self.assertEqual(rotate_params.vault_uri, cluster.properties.disk_encryption_properties.vault_uri)
self.assertEqual(rotate_params.key_name, cluster.properties.disk_encryption_properties.key_name)
self.assertEqual(msi.id.lower(), cluster.properties.disk_encryption_properties.msi_resource_id.lower())
@ResourceGroupPreparer(name_prefix='hdipy-', location=ADLS_LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=ADLS_LOCATION)
def test_create_with_adls_gen1(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-adlsgen1')
create_params = self.get_cluster_create_params_for_adls_gen1(location, cluster_name)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
cluster = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION, kind=Kind.storage_v2)
@StorageAccountPreparer(name_prefix='hdipy2', location=LOCATION, parameter_name='second_storage_account')
def test_create_with_adls_gen2(self, resource_group, location, storage_account, storage_account_key,
second_storage_account, second_storage_account_key):
cluster_name = self.get_resource_name('hdisdk-adlgen2')
create_params = self.get_cluster_create_params_for_adls_gen2(location, cluster_name, storage_account, storage_account_key)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=second_storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=second_storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
cluster = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy2', location=LOCATION, parameter_name='second_storage_account')
def test_create_with_additional_storage(self, resource_group, location, storage_account, storage_account_key,
second_storage_account, second_storage_account_key):
cluster_name = self.get_resource_name('hdisdk-additional')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=second_storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=second_storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
# Add data lake storage gen1 access
create_params = self.get_cluster_create_params_for_adls_gen1(location, cluster_name, create_params)
cluster = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_rserver_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-rserver')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'RServer'
create_params.properties.compute_profile.roles.append(
Role(
name="edgenode",
target_instance_count=1,
hardware_profile=HardwareProfile(vm_size="Standard_D4_v2"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
)
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_mlservices_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-mlservices')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_version="3.6"
create_params.properties.cluster_definition.kind = 'MLServices'
create_params.properties.compute_profile.roles.append(
Role(
name="edgenode",
target_instance_count=1,
hardware_profile=HardwareProfile(vm_size="Standard_D4_v2"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
)
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_list_clusters_in_resource_group(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name1 = self.get_resource_name("hdisdk-cluster-rg1")
cluster_name2 = self.get_resource_name("hdisdk-cluster-rg2")
cluster_list = list(self.hdinsight_client.clusters.list_by_resource_group(rg_name))
self.assertFalse(any(c.name == cluster_name1 for c in cluster_list))
self.assertFalse(any(c.name == cluster_name2 for c in cluster_list))
create_params1 = self.get_cluster_create_params(location, cluster_name1, storage_account, storage_account_key)
create_params2 = self.get_cluster_create_params(location, cluster_name2, storage_account, storage_account_key)
self.hdinsight_client.clusters.create(resource_group.name, cluster_name1, create_params1).wait()
self.hdinsight_client.clusters.create(resource_group.name, cluster_name2, create_params2).wait()
cluster_list = list(self.hdinsight_client.clusters.list_by_resource_group(rg_name))
self.assertTrue(any(c.name == cluster_name1 for c in cluster_list))
self.assertTrue(any(c.name == cluster_name2 for c in cluster_list))
@unittest.skip('This test case lists all clusters under a subscription. '
'In order to avoid storing those cluster infos in session records, skip it for now.')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_list_clusters_in_subscription(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name1 = self.get_resource_name("hdisdk-cluster")
cluster_name2 = self.get_resource_name("hdisdk-cluster")
cluster_list = list(self.hdinsight_client.clusters.list())
self.assertFalse(any(c.name == cluster_name1 for c in cluster_list))
self.assertFalse(any(c.name == cluster_name2 for c in cluster_list))
create_params1 = self.get_cluster_create_params(location, cluster_name1, storage_account, storage_account_key)
create_params2 = self.get_cluster_create_params(location, cluster_name2, storage_account, storage_account_key)
self.hdinsight_client.clusters.create(resource_group.name, cluster_name1, create_params1).wait()
self.hdinsight_client.clusters.create(resource_group.name, cluster_name2, create_params2).wait()
cluster_list = list(self.hdinsight_client.clusters.list())
self.assertTrue(any(c.name == cluster_name1 for c in cluster_list))
self.assertTrue(any(c.name == cluster_name2 for c in cluster_list))
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_hue_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-applications-hue')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_version="3.6"
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
application_name = "MyApplication"
application = Application(
properties=ApplicationProperties(
install_script_actions=[
RuntimeScriptAction(
name="InstallHue",
uri="https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh",
parameters="-version latest -port 20000",
roles=["edgenode"]
)
],
application_type="CustomApplication",
compute_profile=ComputeProfile(
roles=[
Role(
name="edgenode",
hardware_profile=HardwareProfile(
vm_size="Large"
),
target_instance_count = 1
)
]
)
)
)
self.hdinsight_client.applications.create(resource_group.name, cluster_name, application_name, application).wait()
application_list = list(self.hdinsight_client.applications.list_by_cluster(resource_group.name, cluster_name))
self.assertGreater(len(application_list), 0)
application_match = [item for item in application_list if item.name == application_name]
self.assertIsNotNone(application_match)
self.assertEqual(len(application_match), 1)
self.hdinsight_client.applications.delete(resource_group.name, cluster_name, application_name).wait()
application_list = list(self.hdinsight_client.applications.list_by_cluster(resource_group.name, cluster_name))
self.assertEqual(len(application_list), 0)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_get_configurations(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-configs')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
hive_site = 'hive-site'
hive_config = {
'key1': 'value1',
'key2': 'value2'
}
mapred_site = 'mapred-site'
mapred_config = {
'key5': 'value5',
'key6': 'value6'
}
yarn_site = 'yarn-site'
yarn_config = {
'key7': 'value7',
'key8': 'value8'
}
core_site = 'core-site'
gateway = 'gateway'
create_params.properties.cluster_definition.configurations[hive_site] = hive_config
create_params.properties.cluster_definition.configurations[mapred_site] = mapred_config
create_params.properties.cluster_definition.configurations[yarn_site] = yarn_config
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
hive = self.hdinsight_client.configurations.get(rg_name, cluster_name, hive_site)
self.assertEqual(hive, hive_config)
mapred = self.hdinsight_client.configurations.get(rg_name, cluster_name, mapred_site)
self.assertEqual(mapred, mapred_config)
yarn = self.hdinsight_client.configurations.get(rg_name, cluster_name, yarn_site)
self.assertEqual(yarn, yarn_config)
gateway = self.hdinsight_client.configurations.get(rg_name, cluster_name, gateway)
self.assertEqual(len(gateway), 3)
core = self.hdinsight_client.configurations.get(rg_name, cluster_name, core_site)
self.assertEqual(len(core), 2)
self.assertTrue('fs.defaultFS' in core)
storage_key_prefix = 'fs.azure.account.key.'
self.assertTrue(any(key.startswith(storage_key_prefix) for key in core))
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_http_extended(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-http')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
gateway = 'gateway'
user_name = self.cluster_username
user_password = self.cluster_password
http_settings = self.hdinsight_client.configurations.get(rg_name, cluster_name, gateway)
self.validate_http_settings(http_settings, user_name, user_password)
new_password = '******'
update_params = {
'restAuthCredential.isEnabled': 'true',
'restAuthCredential.username': user_name,
'restAuthCredential.password': new_password
}
self.hdinsight_client.configurations.update(rg_name, cluster_name, gateway, update_params).wait()
http_settings = self.hdinsight_client.configurations.get(rg_name, cluster_name, gateway)
self.validate_http_settings(http_settings, user_name, new_password)
def test_get_usages(self):
usages = self.hdinsight_client.locations.list_usages(LOCATION)
self.assertIsNotNone(usages)
self.assertIsNotNone(usages.value)
for usage in usages.value:
self.assertIsNotNone(usage)
self.assertIsNotNone(usage.current_value)
self.assertIsNotNone(usage.limit)
self.assertIsNotNone(usage.name)
self.assertIsNotNone(usage.unit)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_oms_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-oms')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Spark'
create_params.properties.cluster_version="3.6"
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
self.hdinsight_client.extensions.enable_monitoring(rg_name, cluster_name, self.workspace_id).wait()
monitoring_status = self.hdinsight_client.extensions.get_monitoring_status(rg_name, cluster_name)
self.assertTrue(monitoring_status.cluster_monitoring_enabled)
self.assertEqual(monitoring_status.workspace_id, self.workspace_id)
self.hdinsight_client.extensions.disable_monitoring(rg_name, cluster_name).wait()
monitoring_status = self.hdinsight_client.extensions.get_monitoring_status(rg_name, cluster_name)
self.assertFalse(monitoring_status.cluster_monitoring_enabled)
self.assertIsNone(monitoring_status.workspace_id)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_resize_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-clusterresize')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
workernode_params = next(item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
workernode = next(item for item in cluster.properties.compute_profile.roles if item.name == 'workernode')
self.assertEqual(workernode_params.target_instance_count, workernode.target_instance_count)
self.hdinsight_client.clusters.resize(rg_name, cluster_name, workernode_params.target_instance_count + 1).wait()
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
workernode = next(item for item in cluster.properties.compute_profile.roles if item.name == 'workernode')
self.assertEqual(workernode_params.target_instance_count + 1, workernode.target_instance_count)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_script_actions_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-scriptactions')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
install_giraph = "https://hdiconfigactions.blob.core.windows.net/linuxgiraphconfigactionv01/giraph-installer-v01.sh"
script_name = "script1"
# Execute script actions, and persist on success.
script_action_params = self.get_execute_script_action_params(script_name, install_giraph)
self.hdinsight_client.clusters.execute_script_actions(rg_name, cluster_name, True, script_action_params).wait()
# List script actions and validate script is persisted.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(script_action_list))
script_action = script_action_list[0]
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
# Delete script action.
self.hdinsight_client.script_actions.delete(rg_name, cluster_name, script_name)
# List script actions and validate script is deleted.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(0, len(script_action_list))
# List script action history and validate script appears there.
list_history_response = list(self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(list_history_response))
script_action = list_history_response[0]
self.assertEqual(1, len(script_action.execution_summary))
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
self.assertEqual("Succeeded", script_action.status)
# Get the script action by ID and validate it's the same action.
script_action = self.hdinsight_client.script_actions.get_execution_detail(rg_name, cluster_name, str(list_history_response[0].script_execution_id))
self.assertEqual(script_action_params[0].name, script_action.name)
# Execute script actions, but don't persist on success.
script_action_params = self.get_execute_script_action_params("script5baf", install_giraph)
self.hdinsight_client.clusters.execute_script_actions(rg_name, cluster_name, False, script_action_params).wait()
# List script action history and validate the new script also appears.
list_history_response = list(self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(2, len(list_history_response))
script_action = next(a for a in list_history_response if a.name == script_action_params[0].name)
self.assertIsNotNone(script_action)
self.assertEqual(1, len(script_action.execution_summary))
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
self.assertEqual("Succeeded", script_action.status)
# Promote non-persisted script.
self.hdinsight_client.script_execution_history.promote(rg_name, cluster_name, str(list_history_response[0].script_execution_id))
# List script action list and validate the promoted script is the only one there.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(script_action_list))
script_action = script_action_list[0]
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
# List script action history and validate all three scripts are there.
list_history_response = list(self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(2, len(list_history_response))
self.assertTrue(all(a.status == "Succeeded" for a in list_history_response))
def get_execute_script_action_params(self, script_name, script_uri):
return [
RuntimeScriptAction(
name = script_name,
uri = script_uri,
roles = ['headnode', 'workernode']
)
]
def get_cluster_create_params_for_adls_gen1(self, location, cluster_name, create_params=None):
if create_params is None:
create_params = self.get_cluster_create_params(location, cluster_name)
cluster_identity = 'clusterIdentity'
cluster_identity_config = {
'clusterIdentity.applicationId': self.adls_client_id,
'clusterIdentity.certificate': self.cert_content,
'clusterIdentity.aadTenantId': 'https://login.windows.net/{}'.format(self.tenant_id),
'clusterIdentity.resourceUri': 'https://datalake.azure.net/',
'clusterIdentity.certificatePassword': self.cert_password
}
create_params.properties.cluster_definition.configurations[cluster_identity] = cluster_identity_config
is_default = len(create_params.properties.storage_profile.storageaccounts) == 0
if is_default:
core_site = 'core-site'
core_config = {
'fs.defaultFS': 'adl://home',
'dfs.adls.home.hostname': '{}.azuredatalakestore.net'.format(self.adls_account_name),
'dfs.adls.home.mountpoint': self.adls_home_mountpoint
}
create_params.properties.cluster_definition.configurations[core_site] = core_config
return create_params
def get_cluster_create_params_for_adls_gen2(self, location, cluster_name, storage_account, storage_account_key, create_params=None):
if create_params is None:
create_params = self.get_cluster_create_params(location, cluster_name)
is_default = len(create_params.properties.storage_profile.storageaccounts) == 0
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=storage_account.name + STORAGE_ADLS_FILE_SYSTEM_ENDPOINT_SUFFIX,
key=storage_account_key,
file_system=cluster_name.lower(),
is_default= is_default
)
)
return create_params
def get_cluster_create_params(self, location, cluster_name, storage_account=None, storage_account_key=None):
storage_accounts = []
if storage_account:
# Specify storage account details only when storage arguments are provided
storage_accounts.append(
StorageAccount(
name=storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=storage_account_key,
container=cluster_name.lower(),
is_default=True
)
)
return ClusterCreateParametersExtended(
location=location,
tags={},
properties=ClusterCreateProperties(
cluster_version="3.6",
os_type=OSType.linux,
tier=Tier.standard,
cluster_definition=ClusterDefinition(
kind="hadoop",
configurations={
"gateway": {
"restAuthCredential.isEnabled": "true",
"restAuthCredential.username": self.cluster_username,
"restAuthCredential.password": self.cluster_password
}
}
),
compute_profile=ComputeProfile(
roles=[
Role(
name="headnode",
target_instance_count=2,
hardware_profile=HardwareProfile(vm_size="Large"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
),
Role(
name="workernode",
target_instance_count=3,
hardware_profile=HardwareProfile(vm_size="Large"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
),
Role(
name="zookeepernode",
target_instance_count=3,
hardware_profile=HardwareProfile(vm_size="Small"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
]
),
storage_profile=StorageProfile(
storageaccounts=storage_accounts
)
)
)
def validate_cluster(self, cluster_name, create_parameters, cluster_response):
self.assertEqual(cluster_name, cluster_response.name)
self.assertEqual(create_parameters.properties.tier, cluster_response.properties.tier)
self.assertIsNotNone(cluster_response.etag)
self.assertTrue(cluster_response.id.endswith(cluster_name))
self.assertEqual("Running", cluster_response.properties.cluster_state)
self.assertEqual("Microsoft.HDInsight/clusters", cluster_response.type)
self.assertEqual(create_parameters.location, cluster_response.location)
self.assertEqual(create_parameters.tags, cluster_response.tags)
self.assertEqual(1, len([endpoint for endpoint in cluster_response.properties.connectivity_endpoints
if endpoint.name == "HTTPS"]))
self.assertEqual(1, len([endpoint for endpoint in cluster_response.properties.connectivity_endpoints
if endpoint.name == "SSH"]))
self.assertEqual(create_parameters.properties.os_type, cluster_response.properties.os_type)
self.assertIsNone(cluster_response.properties.errors)
self.assertEqual(HDInsightClusterProvisioningState.succeeded, cluster_response.properties.provisioning_state)
self.assertEqual(create_parameters.properties.cluster_definition.kind, cluster_response.properties.cluster_definition.kind)
self.assertEqual(create_parameters.properties.cluster_version, cluster_response.properties.cluster_version[0:3])
self.assertIsNone(cluster_response.properties.cluster_definition.configurations)
def validate_http_settings(self, http_settings, expected_user_name, expected_user_password):
self.assertIsNotNone(http_settings)
self.assertEqual('true', http_settings['restAuthCredential.isEnabled'])
self.assertEqual(expected_user_name, http_settings['restAuthCredential.username'])
self.assertEqual(expected_user_password, http_settings['restAuthCredential.password'])
def _setup_scrubber(self):
super(MgmtHDInsightTest, self)._setup_scrubber()
constants_to_scrub = ['HDI_ADLS_ACCOUNT_NAME', 'HDI_ADLS_CLIENT_ID']
for key in constants_to_scrub:
if hasattr(self.settings, key) and hasattr(self._fake_settings, key):
self.scrubber.register_name_pair(getattr(self.settings, key),
getattr(self._fake_settings, key))
class AzureRMKeyVaultKey(AzureRMModuleBase):
''' Module that creates or deletes keys in Azure KeyVault '''
def __init__(self):
self.module_arg_spec = dict(
key_name=dict(type='str', required=True),
keyvault_uri=dict(type='str', required=True),
pem_file=dict(type='str'),
pem_password=dict(type='str'),
byok_file=dict(type='str'),
state=dict(type='str', default='present', choices=['present', 'absent'])
)
self.results = dict(
changed=False,
state=dict()
)
self.key_name = None
self.keyvault_uri = None
self.pem_file = None
self.pem_password = None
self.state = None
self.client = None
self.tags = None
required_if = [
('pem_password', 'present', ['pem_file'])
]
super(AzureRMKeyVaultKey, self).__init__(self.module_arg_spec,
supports_check_mode=True,
required_if=required_if,
supports_tags=True)
def exec_module(self, **kwargs):
for key in list(self.module_arg_spec.keys()) + ['tags']:
setattr(self, key, kwargs[key])
# Create KeyVaultClient
self.client = KeyVaultClient(self.azure_credentials)
results = dict()
changed = False
try:
results['key_id'] = self.get_key(self.key_name)
# Key exists and will be deleted
if self.state == 'absent':
changed = True
except KeyVaultErrorException:
# Key doesn't exist
if self.state == 'present':
changed = True
self.results['changed'] = changed
self.results['state'] = results
if not self.check_mode:
# Create key
if self.state == 'present' and changed:
results['key_id'] = self.create_key(self.key_name, self.tags)
self.results['state'] = results
self.results['state']['status'] = 'Created'
# Delete key
elif self.state == 'absent' and changed:
results['key_id'] = self.delete_key(self.key_name)
self.results['state'] = results
self.results['state']['status'] = 'Deleted'
else:
if self.state == 'present' and changed:
self.results['state']['status'] = 'Created'
elif self.state == 'absent' and changed:
self.results['state']['status'] = 'Deleted'
return self.results
def get_key(self, name, version=''):
''' Gets an existing key '''
key_bundle = self.client.get_key(self.keyvault_uri, name, version)
if key_bundle:
key_id = KeyVaultId.parse_key_id(key_bundle.key.kid)
return key_id.id
def create_key(self, name, tags, kty='RSA'):
''' Creates a key '''
key_bundle = self.client.create_key(self.keyvault_uri, name, kty, tags=tags)
key_id = KeyVaultId.parse_key_id(key_bundle.key.kid)
return key_id.id
def delete_key(self, name):
''' Deletes a key '''
deleted_key = self.client.delete_key(self.keyvault_uri, name)
key_id = KeyVaultId.parse_key_id(deleted_key.key.kid)
return key_id.id
def import_key(self, key_name, destination=None, key_ops=None, disabled=False, expires=None,
not_before=None, tags=None, pem_file=None, pem_password=None, byok_file=None):
""" Import a private key. Supports importing base64 encoded private keys from PEM files.
Supports importing BYOK keys into HSM for premium KeyVaults. """
def _to_bytes(hex_string):
# zero pads and decodes a hex string
if len(hex_string) % 2:
hex_string = '{0}'.format(hex_string)
return codecs.decode(hex_string, 'hex_codec')
def _set_rsa_parameters(dest, src):
# map OpenSSL parameter names to JsonWebKey property names
conversion_dict = {
'modulus': 'n',
'publicExponent': 'e',
'privateExponent': 'd',
'prime1': 'p',
'prime2': 'q',
'exponent1': 'dp',
'exponent2': 'dq',
'coefficient': 'qi'
}
# regex: looks for matches that fit the following patterns:
# integerPattern: 65537 (0x10001)
# hexPattern:
# 00:a0:91:4d:00:23:4a:c6:83:b2:1b:4c:15:d5:be:
# d8:87:bd:c9:59:c2:e5:7a:f5:4a:e7:34:e8:f0:07:
# The desired match should always be the first component of the match
regex = re.compile(r'([^:\s]*(:[^\:)]+\))|([^:\s]*(:\s*[0-9A-Fa-f]{2})+))')
# regex2: extracts the hex string from a format like: 65537 (0x10001)
regex2 = re.compile(r'(?<=\(0x{1})([0-9A-Fa-f]*)(?=\))')
key_params = crypto.dump_privatekey(crypto.FILETYPE_TEXT, src).decode('utf-8')
for match in regex.findall(key_params):
comps = match[0].split(':', 1)
name = conversion_dict.get(comps[0], None)
if name:
value = comps[1].replace(' ', '').replace('\n', '').replace(':', '')
try:
value = _to_bytes(value)
except Exception: # pylint:disable=broad-except
# if decoding fails it is because of an integer pattern. Extract the hex
# string and retry
value = _to_bytes(regex2.findall(value)[0])
setattr(dest, name, value)
key_attrs = KeyAttributes(not disabled, not_before, expires)
key_obj = JsonWebKey(key_ops=key_ops)
if pem_file:
key_obj.kty = 'RSA'
with open(pem_file, 'r') as f:
pem_data = f.read()
# load private key and prompt for password if encrypted
try:
pem_password = str(pem_password).encode() if pem_password else None
# despite documentation saying password should be a string, it needs to actually
# be UTF-8 encoded bytes
pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, pem_data, pem_password)
except crypto.Error:
pass # wrong password
except TypeError:
pass # no pass provided
_set_rsa_parameters(key_obj, pkey)
elif byok_file:
with open(byok_file, 'rb') as f:
byok_data = f.read()
key_obj.kty = 'RSA-HSM'
key_obj.t = byok_data
return self.client.import_key(
self.keyvault_uri, key_name, key_obj, destination == 'hsm', key_attrs, tags)
