class Vault:
def __init__(self):
self.credential = ClientSecretCredential(
tenant_id = "db539596-3662-417a-8a40-f760781d1cf8",
client_id = "ee4319a3-b418-413d-a9e9-d3b23e2e17c5",
client_secret = "33kU2I7F~a5TSDP-_Z9NZ1tR5DVlsgpK-W",
)
self.secret_client = SecretClient(vault_url="https://{0}.vault.azure.net".format(os.getenv(("KEY_VAULT_NAME"))),credential=self.credential)
self.key_client = KeyClient(vault_url="https://{0}.vault.azure.net".format(os.getenv(("KEY_VAULT_NAME"))),credential=self.credential)
self.key_ops = ["encrypt", "decrypt", "sign", "verify", "wrapKey", "unwrapKey"]
def get_secret(self,key):
return self.secret_client.get_secret(key).value
def set_secret(self,key,value):
self.secret_client.set_secret(key,value)
def set_key(self,key_name,key_size,key_ops):
self.key_client.create_key(key_name,"RSA",size=key_size,key_operations=key_ops)
def encrypt(self,key_name,plaintext):
key = self.key_client.get_key(key_name)
crypto_client = CryptographyClient(key,credential=self.credential)
text = crypto_client.encrypt(EncryptionAlgorithm.rsa_oaep,bytes(plaintext.encode()))
return text.ciphertext
def decrypt(self,ciphertext,key_name):
key = self.key_client.get_key(key_name)
crypto_client = CryptographyClient(key,credential=self.credential)
text = crypto_client.decrypt(EncryptionAlgorithm.rsa_oaep,ciphertext)
return text.plaintext.decode()
def deleted_key_recovery(self):
"""
a sample of enumerating, retrieving, recovering and purging deleted keys from a key vault
"""
# create a vault enabling the soft delete feature
vault = self.create_vault()
# create a key client
credential = DefaultAzureCredential()
key_client = KeyClient(vault_url=vault.properties.vault_uri,
credential=credential)
# create keys in the vault
key_to_recover = get_name('key')
key_to_purge = get_name('key')
key = key_client.create_key(key_to_recover, 'RSA')
print('created key {}'.format(key.name))
key = key_client.create_key(key_to_purge, 'RSA')
print('created key {}'.format(key.name))
# list the vault keys
keys = key_client.list_properties_of_keys()
print("keys:")
for key in keys:
print(key.name)
# delete the keys
key_client.begin_delete_key(key_to_recover).wait()
print('deleted key {}'.format(key_to_recover))
key_client.begin_delete_key(key_to_purge).wait()
print('deleted key {}'.format(key_to_purge))
# list the deleted keys
deleted_keys = key_client.list_deleted_keys()
print("deleted keys:")
for deleted_key in deleted_keys:
print(deleted_key.name)
# recover a deleted key
recover_key_poller = key_client.begin_recover_deleted_key(
key_to_recover)
recovered_key = recover_key_poller.result()
print('recovered key {}'.format(recovered_key.name))
# purge a deleted key
key_client.purge_deleted_key(key_to_purge)
time.sleep(50)
print('purged key {}'.format(key_to_purge))
# list the vaults key
keys = key_client.list_properties_of_keys()
print("all of the keys in the client's vault:")
for key in keys:
print(key.name)
class Vault:
def __init__(self):
if os.environ.get('IS_PROD', None):
self.credential = ClientSecretCredential(
tenant_id=os.environ.get('tenant_id', None),
client_id=os.environ.get('client_id', None),
client_secret=os.environ.get('client_secret', None),
)
self.secret_client = SecretClient(vault_url=os.environ.get(
'vault_url', None),
credential=self.credential)
self.key_client = KeyClient(vault_url=os.environ.get(
'vault_url', None),
credential=self.credential)
else:
self.credential = ClientSecretCredential(
tenant_id=Configuration.tenant_id,
client_id=Configuration.client_id,
client_secret=Configuration.client_secret,
)
self.secret_client = SecretClient(
vault_url=Configuration.vault_url, credential=self.credential)
self.key_client = KeyClient(vault_url=Configuration.vault_url,
credential=self.credential)
self.key_ops = [
"encrypt", "decrypt", "sign", "verify", "wrapKey", "unwrapKey"
]
def get_secret(self, key):
return self.secret_client.get_secret(key).value
def set_secret(self, key, value):
self.secret_client.set_secret(key, value)
def set_key(self, key_name, key_size, key_ops):
self.key_client.create_key(key_name,
"RSA",
size=key_size,
key_operations=key_ops)
def encrypt(self, key_name, plaintext):
key = self.key_client.get_key(key_name)
crypto_client = CryptographyClient(key, credential=self.credential)
text = crypto_client.encrypt(EncryptionAlgorithm.rsa_oaep,
bytes(plaintext.encode()))
return text.ciphertext
def decrypt(self, ciphertext, key_name):
key = self.key_client.get_key(key_name)
crypto_client = CryptographyClient(key, credential=self.credential)
text = crypto_client.decrypt(EncryptionAlgorithm.rsa_oaep, ciphertext)
return text.plaintext.decode()
def close_all_connections(self):
self.secret_client.close()
self.key_client.close()
def create_key(self, group_name, location, key_vault, tenant_id,
object_id):
if self.is_live:
result = self.keyvault_client.vaults.begin_create_or_update(
group_name, key_vault, {
'location': location,
'properties': {
'sku': {
'family': "A",
'name': 'standard'
},
'tenant_id':
tenant_id,
"access_policies": [{
"tenant_id": tenant_id,
"object_id": object_id,
"permissions": {
"keys": [
"encrypt", "decrypt", "wrapKey",
"unwrapKey", "sign", "verify", "get",
"list", "create", "update", "import",
"delete", "backup", "restore", "recover",
"purge"
]
}
}],
'enabled_for_disk_encryption':
True,
}
}).result()
vault_url = result.properties.vault_uri
vault_id = result.id
from azure.keyvault.keys import KeyClient
credentials = self.settings.get_azure_core_credentials()
key_client = KeyClient(vault_url, credentials)
# [START create_key]
from dateutil import parser as date_parse
expires_on = date_parse.parse("2050-02-02T08:00:00.000Z")
key = key_client.create_key("testkey",
"RSA",
size=2048,
expires_on=expires_on)
return (vault_id, key.id)
else:
return ('000', '000')
def backup_restore_key(self):
"""
backs up a key vault key and restores it to another key vault
"""
# create a key vault
first_vault = self.create_vault()
# create a key client
credential = DefaultAzureCredential()
first_key_client = KeyClient(
vault_url=first_vault.properties.vault_uri, credential=credential)
# create a key in the vault
key_name = get_name('key')
key = first_key_client.create_key(key_name, 'RSA')
print('created key {}'.format(key.name))
# list the keys in the vault
keys = first_key_client.list_properties_of_keys()
print("keys in the first vault:")
for key in keys:
print(key.name)
# backup the key
backup = first_key_client.backup_key(key_name)
print('backed up key {}'.format(key_name))
# create a second vault
second_vault = self.create_vault()
# create a key client
second_key_client = KeyClient(
vault_url=second_vault.properties.vault_uri, credential=credential)
# restore the key to the new vault
restored = second_key_client.restore_key_backup(backup)
print('restored secret {}'.format(restored.name))
# list the keys in the new vault
keys = second_key_client.list_properties_of_keys()
print("keys in the second vault:")
for key in keys:
print(key.name)
def run_sample():
# Instantiate a key client that will be used to call the service.
# Notice that the client is using default Azure credentials.
# To make default credentials work, ensure that environment variables 'AZURE_CLIENT_ID',
# 'AZURE_CLIENT_SECRET' and 'AZURE_TENANT_ID' are set with the service principal credentials.
VAULT_URL = os.environ["VAULT_URL"]
credential = DefaultAzureCredential()
client = KeyClient(vault_url=VAULT_URL, credential=credential)
try:
# Let's create a Key of type RSA.
# if the key already exists in the Key Vault, then a new version of the key is created.
print("\n1. Create Key")
key = client.create_key("keyName", "RSA")
print("Key with name '{0}' created with key type '{1}'".format(
key.name, key.key_material.kty))
# Backups are good to have, if in case keys gets deleted accidentally.
# For long term storage, it is ideal to write the backup to a file.
print("\n1. Create a backup for an existing Key")
key_backup = client.backup_key(key.name)
print("Backup created for key with name '{0}'.".format(key.name))
# The rsa key is no longer in use, so you delete it.
client.delete_key(key.name)
# To ensure key is deleted on the server side.
print("\nDeleting key...")
time.sleep(20)
print("Deleted Key with name '{0}'".format(key.name))
# In future, if the key is required again, we can use the backup value to restore it in the Key Vault.
print("\n2. Restore the key using the backed up key bytes")
key = client.restore_key(key_backup)
print("Restored Key with name '{0}'".format(key.name))
except HttpResponseError as e:
print("\nrun_sample has caught an error. {0}".format(e.message))
finally:
print("\nrun_sample done")
#
# 5. Restore a key (restore_key_backup)
# ----------------------------------------------------------------------------------------------------------
# Instantiate a key client that will be used to call the service.
# Notice that the client is using default Azure credentials.
# To make default credentials work, ensure that environment variables 'AZURE_CLIENT_ID',
# 'AZURE_CLIENT_SECRET' and 'AZURE_TENANT_ID' are set with the service principal credentials.
VAULT_URL = os.environ["VAULT_URL"]
credential = DefaultAzureCredential()
client = KeyClient(vault_url=VAULT_URL, credential=credential)
# Let's create a Key of type RSA.
# if the key already exists in the Key Vault, then a new version of the key is created.
print("\n.. Create Key")
key = client.create_key("keyName", "RSA")
print("Key with name '{0}' created with key type '{1}'".format(
key.name, key.key_type))
# Backups are good to have, if in case keys gets deleted accidentally.
# For long term storage, it is ideal to write the backup to a file.
print("\n.. Create a backup for an existing Key")
key_backup = client.backup_key(key.name)
print("Backup created for key with name '{0}'.".format(key.name))
# The rsa key is no longer in use, so you delete it.
print("\n.. Delete the key")
delete_operation = client.begin_delete_key(key.name)
deleted_key = delete_operation.result()
print("Deleted key with name '{0}'".format(deleted_key.name))
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
TENANT_ID = os.environ.get("AZURE_TENANT_ID", None)
DISK_ENCRYPTION_SET_NAME = "diskencryptionsetxxx"
GROUP_NAME = "testgroupx"
KEY_VAULT = "keyvaultxyzxxxxx"
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
compute_client = ComputeManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
keyvault_client = KeyVaultManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
# Create resource group
resource_client.resource_groups.create_or_update(
GROUP_NAME,
{"location": "eastus"}
)
# Create key
vault = keyvault_client.vaults.begin_create_or_update(
GROUP_NAME,
KEY_VAULT,
{
'location': "eastus",
'properties': {
'sku': {
'name': 'standard',
'family': 'A'
},
'tenant_id': TENANT_ID,
"access_policies": [
{
"tenant_id": TENANT_ID,
"object_id": "123743cc-88ef-49ee-920e-13958fe5697d",
"permissions": {
"keys": [
"encrypt",
"decrypt",
"wrapKey",
"unwrapKey",
"sign",
"verify",
"get",
"list",
"create",
"update",
"import",
"delete",
"backup",
"restore",
"recover",
"purge"
]
}
}
],
'enabled_for_disk_encryption': True,
}
}
).result()
key_client = KeyClient(vault.properties.vault_uri, DefaultAzureCredential())
expires_on = date_parse.parse("2050-02-02T08:00:00.000Z")
key = key_client.create_key(
"testkey",
"RSA",
size=2048,
expires_on=expires_on
)
# Create disk encryption set
encryption_set = compute_client.disk_encryption_sets.begin_create_or_update(
GROUP_NAME,
DISK_ENCRYPTION_SET_NAME,
{
"location": "eastus",
"identity": {
"type": "SystemAssigned"
},
"active_key": {
"source_vault": {
"id": vault.id
},
"key_url": key.id
}
}
).result()
print("Create disk encryption set:\n{}".format(encryption_set))
# Get disk encrytion set
encryption_set = compute_client.disk_encryption_sets.get(
GROUP_NAME,
DISK_ENCRYPTION_SET_NAME
)
print("Get disk encryption set:\n{}".format(encryption_set))
# Update disk encryption set
encryption_set = compute_client.disk_encryption_sets.begin_update(
GROUP_NAME,
DISK_ENCRYPTION_SET_NAME,
{
"active_key": {
"source_vault": {
"id": vault.id
},
"key_url": key.id
},
"tags": {
"department": "Development",
"project": "Encryption"
}
}
).result()
print("Update disk encryption set:\n{}".format(encryption_set))
# Delete disk encryption set
compute_client.disk_encryption_sets.begin_delete(
GROUP_NAME,
DISK_ENCRYPTION_SET_NAME
)
print("Delete disk encryption set.\n")
# Delete Group
resource_client.resource_groups.begin_delete(
GROUP_NAME
).result()
def create_key(self, group_name, location, key_vault, tenant_id, object_id,
object_id_2):
if self.is_live:
result = self.keyvault_client.vaults.begin_create_or_update(
group_name, key_vault, {
'location': location,
'properties': {
'sku': {
'family': 'A',
'name': 'standard'
},
'tenant_id':
tenant_id,
"access_policies": [{
"tenant_id": tenant_id,
"object_id": object_id,
"permissions": {
"keys": [
"get", "create", "delete", "list",
"update", "import", "backup", "restore",
"recover"
],
"secrets": [
"get", "list", "set", "delete", "backup",
"restore", "recover"
],
"certificates": [
"get", "list", "delete", "create",
"import", "update", "managecontacts",
"getissuers", "listissuers", "setissuers",
"deleteissuers", "manageissuers", "recover"
],
"storage": [
"get", "list", "delete", "set", "update",
"regeneratekey", "setsas", "listsas",
"getsas", "deletesas"
]
}
}, {
"tenantId": tenant_id,
"objectId": object_id_2,
"permissions": {
"keys":
["unwrapKey", "get", "wrapKey", "list"]
}
}],
'enabled_for_disk_encryption':
True,
'enable_soft_delete':
True,
'soft_delete_retention_in_days':
90,
'nework_acls': {
'bypass': '******',
'default_action': "Allow",
'ip_rules': [],
'virtual_network_rules': []
}
}
}).result()
vault_url = result.properties.vault_uri
vault_id = result.id
from azure.keyvault.keys import KeyClient
credentials = self.settings.get_azure_core_credentials()
key_client = KeyClient(vault_url, credentials)
# [START create_key]
from dateutil import parser as date_parse
expires_on = date_parse.parse("2050-02-02T08:00:00.000Z")
key = key_client.create_key("testkey",
"RSA",
size=2048,
expires_on=expires_on)
return (vault_id, key.id)
else:
return ('000', '000')
class MgmtHDInsightTest(AzureMgmtTestCase):
def setUp(self):
super(MgmtHDInsightTest, self).setUp()
# create client
self.hdinsight_client = self.create_mgmt_client(HDInsightManagementClient)
self.msi_client = self.create_mgmt_client(ManagedServiceIdentityClient)
self.vault_mgmt_client = self.create_mgmt_client(KeyVaultManagementClient)
# sensitive test configs
self.tenant_id = self.settings.TENANT_ID
self.adls_account_name = self.settings.HDI_ADLS_ACCOUNT_NAME
self.adls_client_id = self.settings.HDI_ADLS_CLIENT_ID
# Non-sensitive test configs
self.cluster_username = '******'
self.cluster_password = '******'
self.ssh_username = '******'
self.ssh_password = '******'
self.adls_home_mountpoint = '/clusters/hdi'
self.cert_password = '******'
self.cert_content = 'MIIE5TCCAs0CFDGDR+Zz5To67BBDmsF3agvAtdUvMA0GCSqGSIb3DQEBCwUAMC8xCzAJBgNVBAYTAmNuMQ4wDAYDVQQIDAVjaGluYTEQMA4GA1UECgwHenp5dGVzdDAeFw0yMDA0MTYxMDEzNTlaFw0yMTA0MTYxMDEzNTlaMC8xCzAJBgNVBAYTAmNuMQ4wDAYDVQQIDAVjaGluYTEQMA4GA1UECgwHenp5dGVzdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMmvOHmbbD5kIXnvIdEGFPVd3dHe8RFHBKMBSyqVdcwiijPsaQGkx2/PHyD019FTFPDcwo5Wt2rPXdkOm5Em7NiHcYWjZTF/UDZytf+B2pihI1jDz5qUGf65vIGWI5VVtuuXHSGMOZ4RGyM8/3SyMfAy02db6bJnat2RziZJ2F3FSPfMebGYgFn8uxfYKLGQj7mPEqVyGHxcx0KqbH+2jfzBqDjbaQfo7vBsqxvMfG+2G9SAHaiZPBveym4HuyDDtBZ3ChslVHitWUw0qsbPEuR7Jp8NenAqKVrzvixoepAm7vvErRs/olkJFu3OcIusDIOEQcYem30/rSQejQvrXTcdIvD3UPcqApdzrEZvdiCvIr0ys1Ie2OoOwdz391KCIlH3zgGu4Tdg0vJ+RQIYNT8DNK2PUbbw1bENEcwklgkdT2uIjgh6kOIjMCuy6UzAPnWBF8bl9YY3rB5p/EjnGk8BSp/nCEggMVxzfUhUNQJGKo1C85yFl8Olet/jtPZgf8JV9TOJLdNHlzdkidwrcLGJ8SAEsbveF2w5CtobqeDoClrSieK6ANPPLOWplMNFPZWqUaLe45ReExGYlm7Q+hpdnJem7ywWanJTzI1GoVJMr04ZJmsMEmVYhoSqN4yvFfyWM+rDrKdpzBF6ZqknzWxw+T59u+Eu+PrRMOMAWsDJAgMBAAEwDQYJKoZIhvcNAQELBQADggIBAC/5VPkcsb8sH6wpS3LeKViG2Z+XBZ70Q7FUliE4wfLwP5duzN8DN6eJyxS/nTyxohvI6p18MVgZdhjPcvndR1FE8eBsMqZM0D3eDyCYoiB9Ghm1DSxn0AZyk7/aC4nNYLorZouWa1DdCCdOFZegod5I3USXIvUOBDh9eIQQAGCYdANSLkYsyaZgTaKWiBDH0pTVvCOroCJ7NCayibCMc4vdHUQY/UyKSgOZG0Y2M6AgwNI/yC3tyizu/D8OoF7RUb/A4JqvHnqkY2hGF3/GwEBO4eQrCmHFozrA5qZx40bWP4sXGzcmZz/4Nl9TWRWdIKe5Wh3xz6ZMtJ3gPDwt4Vi59ZHcmq187uwYIRvuvGzD4/oI4zeZ80nxhJUQrZdPh3beaA5GhvTS9MM/RGgkXgB+CA61iF0euAb3FEC3MvpjDwDq9DZDiBulAscRn3QnYhxaL2AxMkNgtj6oaHGx+RlepXKqPd11hsdWhKo8X0zcCVrtrz73b6yDQesSP+lDapLuo/74APJVyAuEXM7zGQSt3zTmeE6RTIywB00VpifqL9HmcekllopPuQrBgs0cpgrs6/VjbC6uwIwV9dUrIsP6geLeocS9j6aQmEIr/E/HjvEZ0kI+03Cw+gQqeAlSeKP6ZY9AgQsCFBIBsgORNcn/Aii0QenXC19LeFC0dJYm'
self.workspace_id = '3741ffb2-a54e-407c-952a-43ab44b57c9d'
self.primary_key = 'qFmud5LfxcCxWUvWcGMhKDp0v0KuBRLsO/AIddX734W7lzdInsVMsB5ILVoOrF+0fCfk/IYYy5SJ9Q+2v4aihQ=='
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-humboldt')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('(BadRequest) Premium Cluster Tier available only for ESP Clusters.')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster_with_premium_tier(self, resource_group, location, storage_account,
storage_account_key):
cluster_name = self.get_resource_name('hdisdk-premium')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.tier = Tier.premium
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_with_empty_extended_parameters(self, resource_group, location, storage_account,
storage_account_key):
cluster_name = self.get_resource_name('hdisdk-cluster')
create_params = ClusterCreateParametersExtended()
# try to create cluster and ensure it throws
try:
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name,
create_params)
cluster = create_poller.result()
self.assertTrue(False, 'should not have made it here')
except Exception:
pass
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_humboldt_cluster_with_custom_vm_sizes(self, resource_group, location, storage_account,
storage_account_key):
cluster_name = self.get_resource_name('hdisdk-customvmsizes')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
headnode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'headnode')
headnode.hardware_profile = HardwareProfile(vm_size="ExtraLarge")
zookeepernode = next(
item for item in create_params.properties.compute_profile.roles if item.name == 'zookeepernode')
zookeepernode.hardware_profile = HardwareProfile(vm_size="Medium")
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_linux_spark_cluster_with_component_version(self, resource_group, location, storage_account,
storage_account_key):
cluster_name = self.get_resource_name('hdisdk-sparkcomponentversions')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Spark'
create_params.properties.cluster_definition.Component_version = {'Spark': '2.2'}
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_kafka_cluster_with_managed_disks(self, resource_group, location, storage_account,
storage_account_key):
cluster_name = self.get_resource_name('hdisdk-kafka')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Kafka'
workernode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
workernode.data_disks_groups = [
DataDisksGroups(
disks_per_node=8
)
]
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('skipping temporarily to unblock azure-keyvault checkin')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
@KeyVaultPreparer(name_prefix='hdipy', location=LOCATION, enable_soft_delete=True)
def test_create_kafka_cluster_with_disk_encryption(self, resource_group, location, storage_account,
storage_account_key, vault):
# create managed identities for Azure resources.
msi_name = self.get_resource_name('hdipyuai')
msi_principal_id = "00000000-0000-0000-0000-000000000000"
msi_id = "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/{}/providers/microsoft.managedidentity/userassignedidentities/{}".format(
resource_group.name, msi_name)
if self.is_live:
msi = self.msi_client.user_assigned_identities.create_or_update(resource_group.name, msi_name, location)
msi_id = msi.id
msi_principal_id = msi.principal_id
# add managed identity to vault
required_permissions = Permissions(
keys=[KeyPermissions.get, KeyPermissions.wrap_key, KeyPermissions.unwrap_key],
secrets=[SecretPermissions.get, SecretPermissions.set, SecretPermissions.delete])
vault.properties.access_policies.append(
AccessPolicyEntry(tenant_id=self.tenant_id,
object_id=msi_principal_id,
permissions=required_permissions)
)
update_params = VaultCreateOrUpdateParameters(location=location,
properties=vault.properties)
vault = self.vault_mgmt_client.vaults.begin_create_or_update(resource_group.name, vault.name,
update_params).result()
self.assertIsNotNone(vault)
# create keyclient
key_client_credential = self.settings.get_azure_core_credentials(scope="https://vault.azure.net/.default")
self.vault_client = KeyClient(vault_url=vault.properties.vault_uri, credential=key_client_credential)
# create key
key_name = self.get_resource_name('hdipykey1')
vault_key = self.vault_client.create_key(key_name, 'RSA')
# create a new key for test rotate
new_key_name = self.get_resource_name('hdipykey2')
new_vault_key = self.vault_client.create_key(new_key_name, 'RSA')
# create HDInsight cluster with Kafka disk encryption
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-kafka-byok')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Kafka'
workernode = next(item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
workernode.data_disks_groups = [
DataDisksGroups(
disks_per_node=8
)
]
create_params.identity = ClusterIdentity(
type=ResourceIdentityType.user_assigned,
user_assigned_identities={msi_id: ClusterIdentityUserAssignedIdentitiesValue()}
)
create_params.properties.disk_encryption_properties = DiskEncryptionProperties(
vault_uri=vault_key.properties.vault_url,
key_name=vault_key.name,
key_version=vault_key.properties.version,
msi_resource_id=msi_id
)
cluster = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
# check disk encryption properties
self.assertIsNotNone(cluster.properties.disk_encryption_properties)
self.assertEqual(create_params.properties.disk_encryption_properties.vault_uri,
cluster.properties.disk_encryption_properties.vault_uri)
self.assertEqual(create_params.properties.disk_encryption_properties.key_name,
cluster.properties.disk_encryption_properties.key_name)
self.assertEqual(create_params.properties.disk_encryption_properties.msi_resource_id.lower(),
cluster.properties.disk_encryption_properties.msi_resource_id.lower())
rotate_params = ClusterDiskEncryptionParameters(
vault_uri=new_vault_key.properties.vault_url,
key_name=new_vault_key.name,
key_version=new_vault_key.properties.version
)
# rotate cluster key
self.hdinsight_client.clusters.rotate_disk_encryption_key(rg_name, cluster_name, rotate_params).wait()
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
# check disk encryption properties
self.assertIsNotNone(cluster.properties.disk_encryption_properties)
self.assertEqual(rotate_params.vault_uri, cluster.properties.disk_encryption_properties.vault_uri)
self.assertEqual(rotate_params.key_name, cluster.properties.disk_encryption_properties.key_name)
self.assertEqual(msi_id.lower(), cluster.properties.disk_encryption_properties.msi_resource_id.lower())
@unittest.skip("There is something wrong in ADLS Gen1 from RP.")
@ResourceGroupPreparer(name_prefix='hdipy-', location=ADLS_LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=ADLS_LOCATION)
def test_create_with_adls_gen1(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-adlsgen1')
create_params = self.get_cluster_create_params_for_adls_gen1(location, cluster_name)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
cluster = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION, kind=Kind.storage_v2)
@StorageAccountPreparer(name_prefix='hdipy2', location=LOCATION, parameter_name='second_storage_account')
def test_create_with_adls_gen2(self, resource_group, location, storage_account, storage_account_key,
second_storage_account, second_storage_account_key):
cluster_name = self.get_resource_name('hdisdk-adlgen2')
create_params = self.get_cluster_create_params_for_adls_gen2(location, cluster_name, storage_account,
storage_account_key)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=second_storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=second_storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
cluster = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip("This test depends on ADLS Gen1. Now there is something wrong with ADLS Gen1.")
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy2', location=LOCATION, parameter_name='second_storage_account')
def test_create_with_additional_storage(self, resource_group, location, storage_account, storage_account_key,
second_storage_account, second_storage_account_key):
cluster_name = self.get_resource_name('hdisdk-additional')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
# Add additional storage account
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=second_storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=second_storage_account_key,
container=cluster_name.lower(),
is_default=False
)
)
# Add data lake storage gen1 access
create_params = self.get_cluster_create_params_for_adls_gen1(location, cluster_name, create_params)
cluster = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params).result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_rserver_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-rserver')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'RServer'
create_params.properties.compute_profile.roles.append(
Role(
name="edgenode",
target_instance_count=1,
hardware_profile=HardwareProfile(vm_size="Standard_D4_v2"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
)
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip("HDInsight will not support to create MLService cluster after 1/1/2021.")
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_create_mlservices_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-mlservices')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_version = "3.6"
create_params.properties.cluster_definition.kind = 'MLServices'
create_params.properties.compute_profile.roles.append(
Role(
name="edgenode",
target_instance_count=1,
hardware_profile=HardwareProfile(vm_size="Standard_D4_v2"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
)
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_list_clusters_in_resource_group(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name1 = self.get_resource_name("hdisdk-cluster-rg1")
cluster_name2 = self.get_resource_name("hdisdk-cluster-rg2")
cluster_list = list(self.hdinsight_client.clusters.list_by_resource_group(rg_name))
self.assertFalse(any(c.name == cluster_name1 for c in cluster_list))
self.assertFalse(any(c.name == cluster_name2 for c in cluster_list))
create_params1 = self.get_cluster_create_params(location, cluster_name1, storage_account, storage_account_key)
create_params2 = self.get_cluster_create_params(location, cluster_name2, storage_account, storage_account_key)
self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name1, create_params1).wait()
self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name2, create_params2).wait()
cluster_list = list(self.hdinsight_client.clusters.list_by_resource_group(rg_name))
self.assertTrue(any(c.name == cluster_name1 for c in cluster_list))
self.assertTrue(any(c.name == cluster_name2 for c in cluster_list))
@unittest.skip('This test case lists all clusters under a subscription. '
'In order to avoid storing those cluster infos in session records, skip it for now.')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_list_clusters_in_subscription(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name1 = self.get_resource_name("hdisdk-cluster")
cluster_name2 = self.get_resource_name("hdisdk-cluster")
cluster_list = list(self.hdinsight_client.clusters.list())
self.assertFalse(any(c.name == cluster_name1 for c in cluster_list))
self.assertFalse(any(c.name == cluster_name2 for c in cluster_list))
create_params1 = self.get_cluster_create_params(location, cluster_name1, storage_account, storage_account_key)
create_params2 = self.get_cluster_create_params(location, cluster_name2, storage_account, storage_account_key)
self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name1, create_params1).wait()
self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name2, create_params2).wait()
cluster_list = list(self.hdinsight_client.clusters.list())
self.assertTrue(any(c.name == cluster_name1 for c in cluster_list))
self.assertTrue(any(c.name == cluster_name2 for c in cluster_list))
@unittest.skip(''' (Conflict) Delete application operation cannot be performed on this cluster at this time as it
is not in 'Running' state. It is possible that the cluster is undergoing other update operations. Please retry later.''')
@ResourceGroupPreparer(name_prefix='hdipy1', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_hue_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
cluster_name = self.get_resource_name('hdisdk-applications-hue')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_version = "3.6"
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
application_name = "MyApplication"
application = Application(
properties=ApplicationProperties(
install_script_actions=[
RuntimeScriptAction(
name="InstallHue",
uri="https://hdiconfigactions.blob.core.windows.net/linuxhueconfigactionv02/install-hue-uber-v02.sh",
parameters="-version latest -port 20000",
roles=["edgenode"]
)
],
application_type="CustomApplication",
compute_profile=ComputeProfile(
roles=[
Role(
name="edgenode",
hardware_profile=HardwareProfile(
vm_size="Large"
),
target_instance_count=1
)
]
)
)
)
self.hdinsight_client.applications.begin_create(resource_group.name, cluster_name, application_name,
application).wait()
application_list = list(self.hdinsight_client.applications.list_by_cluster(resource_group.name, cluster_name))
self.assertGreater(len(application_list), 0)
application_match = [item for item in application_list if item.name == application_name]
self.assertIsNotNone(application_match)
self.assertEqual(len(application_match), 1)
# sleep for robust
import time
time.sleep(120)
self.hdinsight_client.applications.begin_delete(resource_group.name, cluster_name, application_name).wait()
application_list = list(self.hdinsight_client.applications.list_by_cluster(resource_group.name, cluster_name))
self.assertEqual(len(application_list), 0)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_get_configurations(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-configs')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
hive_site = 'hive-site'
hive_config = {
'key1': 'value1',
'key2': 'value2'
}
mapred_site = 'mapred-site'
mapred_config = {
'key5': 'value5',
'key6': 'value6'
}
yarn_site = 'yarn-site'
yarn_config = {
'key7': 'value7',
'key8': 'value8'
}
core_site = 'core-site'
gateway = 'gateway'
create_params.properties.cluster_definition.configurations[hive_site] = hive_config
create_params.properties.cluster_definition.configurations[mapred_site] = mapred_config
create_params.properties.cluster_definition.configurations[yarn_site] = yarn_config
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
hive = self.hdinsight_client.configurations.get(rg_name, cluster_name, hive_site)
self.assertEqual(hive, hive_config)
mapred = self.hdinsight_client.configurations.get(rg_name, cluster_name, mapred_site)
self.assertEqual(mapred, mapred_config)
yarn = self.hdinsight_client.configurations.get(rg_name, cluster_name, yarn_site)
self.assertEqual(yarn, yarn_config)
gateway = self.hdinsight_client.configurations.get(rg_name, cluster_name, gateway)
self.assertEqual(len(gateway), 3)
core = self.hdinsight_client.configurations.get(rg_name, cluster_name, core_site)
self.assertEqual(len(core), 2)
self.assertTrue('fs.defaultFS' in core)
storage_key_prefix = 'fs.azure.account.key.'
self.assertTrue(any(key.startswith(storage_key_prefix) for key in core))
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_gateway_settings(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-http')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
user_name = self.cluster_username
user_password = self.cluster_password
gateway_settings = self.hdinsight_client.clusters.get_gateway_settings(rg_name, cluster_name)
self.validate_gateway_settings(gateway_settings, user_name, user_password)
new_password = '******'
update_params = UpdateGatewaySettingsParameters(is_credential_enabled=True, user_name=user_name,
password=new_password)
self.hdinsight_client.clusters.begin_update_gateway_settings(rg_name, cluster_name, update_params).wait()
gateway_settings = self.hdinsight_client.clusters.get_gateway_settings(rg_name, cluster_name)
self.validate_gateway_settings(gateway_settings, user_name, new_password)
@unittest.skip("hard to test")
def test_get_usages(self):
usages = self.hdinsight_client.locations.list_usages(LOCATION)
self.assertIsNotNone(usages)
self.assertIsNotNone(usages.value)
for usage in usages.value:
self.assertIsNotNone(usage)
self.assertIsNotNone(usage.current_value)
self.assertIsNotNone(usage.limit)
self.assertIsNotNone(usage.name)
self.assertIsNotNone(usage.unit)
@unittest.skip("Script executed failed due to Internal server. The issue is related with RP not SDK.")
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_oms_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-oms')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_params.properties.cluster_definition.kind = 'Spark'
create_params.properties.cluster_version = "3.6"
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
# create log analytics workspace
self.loganalytics_mgmt_client = self.create_mgmt_client(LogAnalyticsManagementClient)
workspace_name = self.get_resource_name('workspaceforpytest')
workspace_params = Workspace(name=workspace_name, location=location)
workspace_poller = self.loganalytics_mgmt_client.workspaces.create_or_update(rg_name, workspace_name,
workspace_params)
workspace = workspace_poller.result()
self.workspace_id = workspace.customer_id
self.primary_key = self.loganalytics_mgmt_client.shared_keys.get_shared_keys(rg_name, workspace_name)
self.hdinsight_client.extensions.enable_monitoring(rg_name, cluster_name, self.workspace_id,
self.primary_key).wait()
monitoring_status = self.hdinsight_client.extensions.get_monitoring_status(rg_name, cluster_name)
self.assertTrue(monitoring_status.cluster_monitoring_enabled)
self.assertEqual(monitoring_status.workspace_id, self.workspace_id)
self.hdinsight_client.extensions.disable_monitoring(rg_name, cluster_name).wait()
monitoring_status = self.hdinsight_client.extensions.get_monitoring_status(rg_name, cluster_name)
self.assertFalse(monitoring_status.cluster_monitoring_enabled)
self.assertIsNone(monitoring_status.workspace_id)
@unittest.skip('(BadRequest) \'targetInstanceCount\' has an invalid value. It must be greater than zero')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_resize_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-clusterresize')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
workernode_params = next(
item for item in create_params.properties.compute_profile.roles if item.name == 'workernode')
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
workernode = next(item for item in cluster.properties.compute_profile.roles if item.name == 'workernode')
self.assertEqual(workernode_params.target_instance_count, workernode.target_instance_count)
self.hdinsight_client.clusters.begin_resize(rg_name, cluster_name,
workernode_params.target_instance_count + 1).wait()
cluster = self.hdinsight_client.clusters.get(rg_name, cluster_name)
workernode = next(item for item in cluster.properties.compute_profile.roles if item.name == 'workernode')
self.assertEqual(workernode_params.target_instance_count + 1, workernode.target_instance_count)
@unittest.skip('hard to test')
@ResourceGroupPreparer(name_prefix='hdipy-', location=LOCATION)
@StorageAccountPreparer(name_prefix='hdipy', location=LOCATION)
def test_script_actions_on_running_cluster(self, resource_group, location, storage_account, storage_account_key):
rg_name = resource_group.name
cluster_name = self.get_resource_name('hdisdk-scriptactions')
create_params = self.get_cluster_create_params(location, cluster_name, storage_account, storage_account_key)
create_poller = self.hdinsight_client.clusters.begin_create(resource_group.name, cluster_name, create_params)
cluster = create_poller.result()
self.validate_cluster(cluster_name, create_params, cluster)
install_giraph = "https://hdiconfigactions.blob.core.windows.net/linuxgiraphconfigactionv01/giraph-installer-v01.sh"
script_name = "script1"
# Execute script actions, and persist on success.
script_action_params = self.get_execute_script_action_params(script_name, install_giraph)
self.hdinsight_client.clusters.begin_execute_script_actions(
rg_name,
cluster_name,
{
"persist_on_success": True,
"script_actions": script_action_params
}
).wait()
# List script actions and validate script is persisted.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(script_action_list))
script_action = script_action_list[0]
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
# Delete script action.
self.hdinsight_client.script_actions.delete(rg_name, cluster_name, script_name)
# List script actions and validate script is deleted.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(0, len(script_action_list))
# List script action history and validate script appears there.
list_history_response = list(
self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(list_history_response))
script_action = list_history_response[0]
self.assertEqual(1, len(script_action.execution_summary))
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
self.assertEqual("Succeeded", script_action.status)
# Get the script action by ID and validate it's the same action.
script_action = self.hdinsight_client.script_actions.get_execution_detail(rg_name, cluster_name, str(
list_history_response[0].script_execution_id))
self.assertEqual(script_action_params[0].name, script_action.name)
# Execute script actions, but don't persist on success.
script_action_params = self.get_execute_script_action_params("script5baf", install_giraph)
self.hdinsight_client.clusters.begin_execute_script_actions(
rg_name,
cluster_name,
{
"persist_on_success": False,
"script_actions": script_action_params
}
).wait()
# List script action history and validate the new script also appears.
list_history_response = list(
self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(2, len(list_history_response))
script_action = next(a for a in list_history_response if a.name == script_action_params[0].name)
self.assertIsNotNone(script_action)
self.assertEqual(1, len(script_action.execution_summary))
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
self.assertEqual("Succeeded", script_action.status)
# Promote non-persisted script.
self.hdinsight_client.script_execution_history.promote(rg_name, cluster_name,
str(list_history_response[0].script_execution_id))
# List script action list and validate the promoted script is the only one there.
script_action_list = list(self.hdinsight_client.script_actions.list_by_cluster(rg_name, cluster_name))
self.assertEqual(1, len(script_action_list))
script_action = script_action_list[0]
self.assertEqual(script_action_params[0].name, script_action.name)
self.assertEqual(script_action_params[0].uri, script_action.uri)
self.assertEqual(script_action_params[0].roles, script_action.roles)
# List script action history and validate all three scripts are there.
list_history_response = list(
self.hdinsight_client.script_execution_history.list_by_cluster(rg_name, cluster_name))
self.assertEqual(2, len(list_history_response))
self.assertTrue(all(a.status == "Succeeded" for a in list_history_response))
def get_execute_script_action_params(self, script_name, script_uri):
return [
RuntimeScriptAction(
name=script_name,
uri=script_uri,
roles=['headnode', 'workernode']
)
]
def get_cluster_create_params_for_adls_gen1(self, location, cluster_name, create_params=None):
if create_params is None:
create_params = self.get_cluster_create_params(location, cluster_name)
cluster_identity = 'clusterIdentity'
cluster_identity_config = {
'clusterIdentity.applicationId': self.adls_client_id,
'clusterIdentity.certificate': self.cert_content,
'clusterIdentity.aadTenantId': 'https://login.windows.net/{}'.format(self.tenant_id),
'clusterIdentity.resourceUri': 'https://datalake.azure.net/',
'clusterIdentity.certificatePassword': self.cert_password
}
create_params.properties.cluster_definition.configurations[cluster_identity] = cluster_identity_config
is_default = len(create_params.properties.storage_profile.storageaccounts) == 0
if is_default:
core_site = 'core-site'
core_config = {
'fs.defaultFS': 'adl://home',
'dfs.adls.home.hostname': '{}.azuredatalakestore.net'.format(self.adls_account_name),
'dfs.adls.home.mountpoint': self.adls_home_mountpoint
}
create_params.properties.cluster_definition.configurations[core_site] = core_config
return create_params
def get_cluster_create_params_for_adls_gen2(self, location, cluster_name, storage_account, storage_account_key,
create_params=None):
if create_params is None:
create_params = self.get_cluster_create_params(location, cluster_name)
is_default = len(create_params.properties.storage_profile.storageaccounts) == 0
create_params.properties.storage_profile.storageaccounts.append(
StorageAccount(
name=storage_account.name + STORAGE_ADLS_FILE_SYSTEM_ENDPOINT_SUFFIX,
key=storage_account_key,
file_system=cluster_name.lower(),
is_default=is_default
)
)
return create_params
def get_cluster_create_params(self, location, cluster_name, storage_account=None, storage_account_key=None):
storage_accounts = []
if storage_account:
# Specify storage account details only when storage arguments are provided
storage_accounts.append(
StorageAccount(
name=storage_account.name + STORAGE_BLOB_SERVICE_ENDPOINT_SUFFIX,
key=storage_account_key,
container=cluster_name.lower(),
is_default=True
)
)
return ClusterCreateParametersExtended(
location=location,
tags={},
properties=ClusterCreateProperties(
cluster_version="3.6",
os_type=OSType.linux,
tier=Tier.standard,
cluster_definition=ClusterDefinition(
kind="hadoop",
configurations={
"gateway": {
"restAuthCredential.isEnabled": "true",
"restAuthCredential.username": self.cluster_username,
"restAuthCredential.password": self.cluster_password
}
}
),
compute_profile=ComputeProfile(
roles=[
Role(
name="headnode",
target_instance_count=2,
hardware_profile=HardwareProfile(vm_size="Large"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
),
Role(
name="workernode",
target_instance_count=3,
hardware_profile=HardwareProfile(vm_size="Large"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
),
Role(
name="zookeepernode",
target_instance_count=3,
hardware_profile=HardwareProfile(vm_size="Small"),
os_profile=OsProfile(
linux_operating_system_profile=LinuxOperatingSystemProfile(
username=self.ssh_username,
password=self.ssh_password
)
)
)
]
),
storage_profile=StorageProfile(
storageaccounts=storage_accounts
)
)
)
def validate_cluster(self, cluster_name, create_parameters, cluster_response):
self.assertEqual(cluster_name, cluster_response.name)
self.assertEqual(create_parameters.properties.tier.lower(), cluster_response.properties.tier.lower())
self.assertIsNotNone(cluster_response.etag)
self.assertTrue(cluster_response.id.endswith(cluster_name))
self.assertEqual("Running", cluster_response.properties.cluster_state)
self.assertEqual("Microsoft.HDInsight/clusters", cluster_response.type)
self.assertEqual(create_parameters.location, cluster_response.location)
self.assertEqual(create_parameters.tags, cluster_response.tags)
self.assertEqual(1, len([endpoint for endpoint in cluster_response.properties.connectivity_endpoints
if endpoint.name == "HTTPS"]))
self.assertEqual(1, len([endpoint for endpoint in cluster_response.properties.connectivity_endpoints
if endpoint.name == "SSH"]))
self.assertEqual(create_parameters.properties.os_type, cluster_response.properties.os_type)
self.assertIsNone(cluster_response.properties.errors)
self.assertEqual(HDInsightClusterProvisioningState.succeeded, cluster_response.properties.provisioning_state)
self.assertEqual(create_parameters.properties.cluster_definition.kind,
cluster_response.properties.cluster_definition.kind)
self.assertEqual(create_parameters.properties.cluster_version, cluster_response.properties.cluster_version[0:3])
self.assertIsNone(cluster_response.properties.cluster_definition.configurations)
def validate_gateway_settings(self, gateway_settings, expected_user_name, expected_user_password):
self.assertIsNotNone(gateway_settings)
self.assertEqual('true', gateway_settings.is_credential_enabled)
self.assertEqual(expected_user_name, gateway_settings.user_name)
self.assertEqual(expected_user_password, gateway_settings.password)
def _setup_scrubber(self):
super(MgmtHDInsightTest, self)._setup_scrubber()
constants_to_scrub = ['HDI_ADLS_ACCOUNT_NAME', 'HDI_ADLS_CLIENT_ID']
for key in constants_to_scrub:
if hasattr(self.settings, key) and hasattr(self._fake_settings, key):
self.scrubber.register_name_pair(getattr(self.settings, key),
getattr(self._fake_settings, key))
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
PASSWORD = os.environ.get("PASSWORD", None)
TENANT_ID = os.environ.get("TENANT_ID", None)
CLIENT_OID = os.environ.get("CLIENT_OID", None)
GROUP_NAME = "testgroupx"
SERVER_KEY = "server_keyxxyyzz"
SERVER = "serverxxy"
VAULT = "vaultxxy"
# Create client
# # For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
sql_client = SqlManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
# - init depended client -
keyvault_client = KeyVaultManagementClient(
credential=DefaultAzureCredential(),
subscription_id=SUBSCRIPTION_ID
)
# - end -
# Create resource group
resource_client.resource_groups.create_or_update(
GROUP_NAME,
{"location": "eastus"}
)
# - init depended resources -
# Create Server
server = sql_client.servers.begin_create_or_update(
GROUP_NAME,
SERVER,
{
"location": "eastus",
"identity": {
"type": "SystemAssigned"
},
"administrator_login": "******",
"administrator_login_password": PASSWORD,
"version": "12.0",
"public_network_access":"Enabled"
}
).result()
print("Create server:\n{}".format(server))
# Create vault
vault = keyvault_client.vaults.begin_create_or_update(
GROUP_NAME,
VAULT,
{
'location': "eastus",
'properties': {
'sku': {
'family': 'A',
'name': 'standard'
},
'tenant_id': TENANT_ID,
"access_policies": [
{
"tenant_id": TENANT_ID,
"object_id": CLIENT_OID,
"permissions": {
"keys": [
"get",
"create",
"delete",
"list",
"update",
"import",
"backup",
"restore",
"recover"
],
"secrets": [
"get",
"list",
"set",
"delete",
"backup",
"restore",
"recover"
],
"certificates": [
"get",
"list",
"delete",
"create",
"import",
"update",
"managecontacts",
"getissuers",
"listissuers",
"setissuers",
"deleteissuers",
"manageissuers",
"recover"
],
"storage": [
"get",
"list",
"delete",
"set",
"update",
"regeneratekey",
"setsas",
"listsas",
"getsas",
"deletesas"
]
}
},
{
"tenantId": TENANT_ID,
"objectId": server.identity.principal_id,
"permissions": {
"keys": [
"unwrapKey",
"get",
"wrapKey",
"list"
]
}
}
],
'enabled_for_disk_encryption': True,
'enable_soft_delete': True,
'soft_delete_retention_in_days': 90,
'nework_acls': {
'bypass': '******',
'default_action': "Allow",
'ip_rules': [],
'virtual_network_rules': []
}
}
}
).result()
key_client = KeyClient(
vault.properties.vault_uri,
DefaultAzureCredential()
)
key = key_client.create_key(
"testkey",
"RSA",
size=2048,
expires_on=date_parse.parse("2050-02-02T08:00:00.000Z")
)
SERVER_KEY = VAULT + "_testkey_" + key.id.split("/")[-1]
# - end -
# Create server key
server_key = sql_client.server_keys.begin_create_or_update(
GROUP_NAME,
SERVER,
SERVER_KEY,
{
# TODO: init resource body
"server_key_type": "AzureKeyVault",
"uri": key.id
}
).result()
print("Create server key:\n{}".format(server_key))
# Get server key
server_key = sql_client.server_keys.get(
GROUP_NAME,
SERVER,
SERVER_KEY
)
print("Get server key:\n{}".format(server_key))
# Delete server key
server_key = sql_client.server_keys.begin_delete(
GROUP_NAME,
SERVER,
SERVER_KEY
).result()
print("Delete server key.\n")
# Delete Group
resource_client.resource_groups.begin_delete(
GROUP_NAME
).result()
