def run_example():
"""Resource Group management example."""
#
# Create the Resource Manager Client with an Application (service principal) token provider
#
subscription_id = os.environ.get(
'AZURE_SUBSCRIPTION_ID',
'11111111-1111-1111-1111-111111111111') # your Azure Subscription Id
credentials = ServicePrincipalCredentials(
client_id=os.environ['AZURE_CLIENT_ID'],
secret=os.environ['AZURE_CLIENT_SECRET'],
tenant=os.environ['AZURE_TENANT_ID'])
resource_client = ResourceManagementClient(credentials, subscription_id)
compute_client = ComputeManagementClient(credentials, subscription_id)
network_client = NetworkManagementClient(credentials, subscription_id)
kv_mgmt_client = KeyVaultManagementClient(credentials, subscription_id)
kv_credentials = ServicePrincipalCredentials(
client_id=os.environ['AZURE_CLIENT_ID'],
secret=os.environ['AZURE_CLIENT_SECRET'],
tenant=os.environ['AZURE_TENANT_ID'],
resource="https://vault.azure.net")
kv_client = KeyVaultClient(kv_credentials)
# Create Resource group
print('\nCreate Resource Group')
resource_group = resource_client.resource_groups.create_or_update(
GROUP_NAME, {'location': LOCATION})
print_item(resource_group)
# Resolve the client_id as object_id for KeyVault access policy.
# If you already know your object_id, you can skip this part
sp_object_id = resolve_service_principal(os.environ['AZURE_CLIENT_ID'])
# Create Key Vault account
print('\nCreate Key Vault account')
async_vault_poller = kv_mgmt_client.vaults.create_or_update(
GROUP_NAME,
KV_NAME,
{
'location': LOCATION,
'properties': {
'sku': {
'name': 'standard'
},
'tenant_id':
os.environ['AZURE_TENANT_ID'],
'access_policies': [{
'tenant_id': os.environ['AZURE_TENANT_ID'],
'object_id': sp_object_id,
'permissions': {
# Only "certificates" and "secrets" are needed for this sample
'certificates': ['all'],
'secrets': ['all']
}
}],
# Critical to allow the VM to download certificates later
'enabled_for_deployment':
True
}
})
vault = async_vault_poller.result()
print_item(vault)
# # KeyVault recommentation is to wait 20 seconds after account creation for DNS update
time.sleep(20)
# Ask KeyVault to create a Certificate
certificate_name = "cert1"
print('\nCreate Key Vault Certificate')
kv_client.create_certificate(vault.properties.vault_uri,
certificate_name,
certificate_policy=DEFAULT_POLICY)
while True:
check = kv_client.get_certificate_operation(vault.properties.vault_uri,
certificate_name)
if check.status != 'inProgress':
break
try:
print("Waiting for certificate creation to finish")
time.sleep(10)
except KeyboardInterrupt:
print("Certificate creation wait cancelled.")
raise
print_item(check)
print('\nGet Key Vault created certificate as a secret')
certificate_as_secret = kv_client.get_secret(
vault.properties.vault_uri,
certificate_name,
"" # Latest version
)
print_item(certificate_as_secret)
print("\nCreate Network")
# Create Network components of the VM
# This is not related to the main topic of this sample and is just required to create the VM
subnet = create_virtual_network(network_client)
public_ip = create_public_ip(network_client)
nic = create_network_interface(network_client, subnet, public_ip)
print_item(nic)
# Create a VM with some Key Vault certificates
params_create = {
'location': LOCATION,
'hardware_profile': get_hardware_profile(),
'network_profile': get_network_profile(nic.id),
'storage_profile': get_storage_profile(),
'os_profile': {
'admin_username':
ADMIN_LOGIN,
'admin_password':
ADMIN_PASSWORD,
'computer_name':
'testkvcertificates',
# This is the Key Vault critical part
'secrets': [{
'source_vault': {
'id': vault.id,
},
'vault_certificates': [{
'certificate_url':
certificate_as_secret.id
}]
}]
}
}
print("\nCreate VM")
vm_poller = compute_client.virtual_machines.create_or_update(
GROUP_NAME,
VM_NAME,
params_create,
)
vm_result = vm_poller.result()
print_item(vm_result)
# Get the PublicIP after VM creation, since assignment is dynamic
public_ip = network_client.public_ip_addresses.get(GROUP_NAME,
PUBLIC_IP_NAME)
print("You can connect to the VM using:")
print("ssh {}@{}".format(
ADMIN_LOGIN,
public_ip.ip_address,
))
print("And password: {}\n".format(ADMIN_PASSWORD))
print("Your certificate is available in this folder: /var/lib/waagent")
print("You must be root to see it (sudo su)\n")
input("Press enter to delete this Resource Group.")
# Delete Resource group and everything in it
print('Delete Resource Group')
delete_async_operation = resource_client.resource_groups.delete(GROUP_NAME)
delete_async_operation.wait()
print("\nDeleted: {}".format(GROUP_NAME))
def run(self):
#----
"""Resource Group management example."""
#
# Create all clients with an Application (service principal) token provider
#
subscription_id = os.environ.get('AZURE_SUBSCRIPTION_ID',
'ef80a466-7372-49e9-b247-57b95886881c'
) # your Azure Subscription Id
credentials = ServicePrincipalCredentials(
client_id='445a1911-819a-41e8-a093-adfd66ca5ccd',
secret='rJ--cHsg@=fucrddh3svx1VUe91q2h1N',
tenant='8ee0f3e4-b788-4efa-bd84-e6bfe7fe9943')
resource_client = ResourceManagementClient(credentials,
subscription_id)
compute_client = ComputeManagementClient(credentials, subscription_id)
storage_client = StorageManagementClient(credentials, subscription_id)
network_client = NetworkManagementClient(credentials, subscription_id)
###########
# Prepare #
###########
# Create Resource group
# print('\nCreate Resource Group')
# resource_client.resource_groups.create_or_update(
# GROUP_NAME, {'location': LOCATION})
# Create a storage account
# print('\nCreate a storage account')
# storage_async_operation = storage_client.storage_accounts.create(
# GROUP_NAME,
# STORAGE_ACCOUNT_NAME,
# {
# 'sku': {'name': 'standard_lrs'},
# 'kind': 'storage',
# 'location': LOCATION
# }
# )
# storage_async_operation.wait()
#
# # Create a NIC
# nic = create_nic(network_client)
#############
# VM Sample #
#############
# Create Linux VM
# print('\nCreating Linux Virtual Machine')
# vm_parameters = create_vm_parameters(nic.id, VM_REFERENCE['linux'])
# async_vm_creation = compute_client.virtual_machines.create_or_update(
# GROUP_NAME, VM_NAME, vm_parameters)
# async_vm_creation.wait()
# Tag the VM
# print('\nTag Virtual Machine')
# async_vm_update = compute_client.virtual_machines.create_or_update(
# GROUP_NAME,
# VM_NAME,
# {
# 'location': LOCATION,
# 'tags': {
# 'who-rocks': 'python',
# 'where': 'on azure'
# }
# }
# )
# async_vm_update.wait()
# Attach data disk
# print('\nAttach Data Disk')
# async_vm_update = compute_client.virtual_machines.create_or_update(
# GROUP_NAME,
# VM_NAME,
# {
# 'location': LOCATION,
# 'storage_profile': {
# 'data_disks': [{
# 'name': 'mydatadisk1',
# 'disk_size_gb': 1,
# 'lun': 0,
# 'vhd': {
# 'uri': "http://{}.blob.core.windows.net/vhds/mydatadisk1.vhd".format(
# STORAGE_ACCOUNT_NAME)
# },
# 'create_option': 'Empty'
# }]
# }
# }
# )
# async_vm_update.wait()
# # Get one the virtual machine by name
# print('\nGet Virtual Machine by Name')
# virtual_machine = compute_client.virtual_machines.get(
# GROUP_NAME,
# VM_NAME
# )
#
# # Detach data disk
# print('\nDetach Data Disk')
# data_disks = virtual_machine.storage_profile.data_disks
# data_disks[:] = [disk for disk in data_disks if disk.name != 'mydatadisk1']
# async_vm_update = compute_client.virtual_machines.create_or_update(
# GROUP_NAME,
# VM_NAME,
# virtual_machine
# )
# virtual_machine = async_vm_update.result()
#
# # Deallocating the VM (resize prepare)
# print('\nDeallocating the VM (resize prepare)')
# async_vm_deallocate = compute_client.virtual_machines.deallocate(
# GROUP_NAME, VM_NAME)
# async_vm_deallocate.wait()
#
# # Update OS disk size by 10Gb
# print('\nUpdate OS disk size')
# # Server is not returning the OS Disk size (None), possible bug in server
# if not virtual_machine.storage_profile.os_disk.disk_size_gb:
# print("\tServer is not returning the OS disk size, possible bug in the server?")
# print("\tAssuming that the OS disk size is 256 GB")
# virtual_machine.storage_profile.os_disk.disk_size_gb = 256
#
# virtual_machine.storage_profile.os_disk.disk_size_gb += 10
# async_vm_update = compute_client.virtual_machines.create_or_update(
# GROUP_NAME,
# VM_NAME,
# virtual_machine
# )
# virtual_machine = async_vm_update.result()
# Start the VM
# print('\nStart VM')
# async_vm_start = compute_client.virtual_machines.start(GROUP_NAME, VM_NAME)
# async_vm_start.wait()
# Restart the VM
# print('\nRestart VM')
# async_vm_restart = compute_client.virtual_machines.restart(
# GROUP_NAME, VM_NAME)
# async_vm_restart.wait()
# Stop the VM
print('\nStop VM')
async_vm_stop = compute_client.virtual_machines.power_off(
GROUP_NAME, VM_NAME)
async_vm_stop.wait()
# List VMs in subscription
# print('\nList VMs in subscription')
# for vm in compute_client.virtual_machines.list_all():
# print("\tVM: {}".format(vm.name))
# List VM in resource group
# print('\nList VMs in resource group')
# for vm in compute_client.virtual_machines.list(GROUP_NAME):
# print("\tVM: {}".format(vm.name))
# Delete VM
# print('\nDelete VM')
# async_vm_delete = compute_client.virtual_machines.delete(
# GROUP_NAME, VM_NAME)
# async_vm_delete.wait()
return
secret=os.environ['AZURE_CLIENT_SECRET'],
tenant=os.environ['AZURE_TENANT_ID']
)
return credentials, subscription_id
# This script expects that the following environment vars are set:
#
# AZURE_TENANT_ID: with your Azure Active Directory tenant id or domain
# AZURE_CLIENT_ID: with your Azure Active Directory Application Client ID
# AZURE_CLIENT_SECRET: with your Azure Active Directory Application Secret
# AZURE_SUBSCRIPTION_ID: with your Azure Subscription Id
#
credentials, subscription_id = get_credentials()
network_client = NetworkManagementClient(credentials, subscription_id)
resource_client = ResourceManagementClient(credentials, subscription_id)
storage_client = StorageManagementClient(credentials, subscription_id)
resource_group_params = {'location':LOCATION}
resource_client.resource_groups.create_or_update(GROUP_NAME, resource_group_params)
vnet_params = { 'location': LOCATION, 'address_space' : { 'address_prefixes': [VNET_AP] } }
async_vnet_creation = network_client.virtual_networks.create_or_update(GROUP_NAME, VNET_NAME, vnet_params)
async_vnet_creation.wait()
ep = ServiceEndpointPropertiesFormat(service='Microsoft.Storage')
ep_list = [ep]
subnet = Subnet(address_prefix = SB_AP, service_endpoints = ep_list)
async_vnet_subnet_creation = network_client.subnets.create_or_update(GROUP_NAME, VNET_NAME, SB_NAME, subnet)
async_vnet_subnet_creation.wait()
if async_vnet_subnet_creation.status() == 'Succeeded':
def network_models(self):
self.log("Getting network models...")
return NetworkManagementClient.models("2017-06-01")
def creating_network_client(self):
return NetworkManagementClient(subscription_id=self.subscriptionId,credentials=self.credential)
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
GROUP_NAME = "testgroupx"
VPN_GATEWAY = "vpn_gatewayxxyyzz"
VIRTUAL_WAN = "virtualwanxxx"
VIRTUAL_HUB = "virtualhubxxxx"
VPN_SITE = "vpnsitexxx"
VPN_SITE_LINK = "vpnSiteLink1"
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
network_client = NetworkManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
# Create resource group
resource_client.resource_groups.create_or_update(GROUP_NAME,
{"location": "eastus"})
# - init depended resources -
# Create virtual wan
network_client.virtual_wans.begin_create_or_update(
GROUP_NAME, VIRTUAL_WAN, {
"location": "West US",
"tags": {
"key1": "value1"
},
"disable_vpn_encryption": False,
"type": "Basic"
}).result()
# Create virtual hub
network_client.virtual_hubs.begin_create_or_update(
GROUP_NAME, VIRTUAL_HUB, {
"location": "West US",
"tags": {
"key1": "value1"
},
"virtual_wan": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/virtualWans/" +
VIRTUAL_WAN + ""
},
"address_prefix": "10.168.0.0/24",
"sku": "Basic"
}).result()
# Create vpn site
network_client.vpn_sites.begin_create_or_update(
GROUP_NAME, VPN_SITE, {
"tags": {
"key1": "value1"
},
"location":
"West US",
"virtual_wan": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/virtualWans/" +
VIRTUAL_WAN + ""
},
"address_space": {
"address_prefixes": ["10.0.0.0/16"]
},
"is_security_site":
False,
"vpn_site_links": [{
"name": VPN_SITE_LINK,
"ip_address": "50.50.50.56",
"link_properties": {
"link_provider_name": "vendor1",
"link_speed_in_mbps": "0"
},
"bgp_properties": {
"bgp_peering_address": "192.168.0.0",
"asn": "1234"
}
}]
}).result()
# - end -
# Create vpn gateway
vpn_gateway = network_client.vpn_gateways.begin_create_or_update(
GROUP_NAME, VPN_GATEWAY, {
"location":
"West US",
"tags": {
"key1": "value1"
},
"virtual_hub": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/virtualHubs/" +
VIRTUAL_HUB + ""
},
"connections": [{
"name":
"vpnConnection1",
"remote_vpn_site": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/vpnSites/" +
VPN_SITE + ""
},
"vpn_link_connections": [{
"name": "Connection-Link1",
"vpn_site_link": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID +
"/resourceGroups/" + GROUP_NAME +
"/providers/Microsoft.Network/vpnSites/" + VPN_SITE +
"/vpnSiteLinks/" + VPN_SITE_LINK + ""
},
"connection_bandwidth": "200",
"vpn_connection_protocol_type": "IKEv2",
"shared_key": "key"
}]
}],
"bgp_settings": {
"asn": "65515",
"peer_weight": "0"
}
}).result()
print("Create vpn gateway:\n{}".format(vpn_gateway))
# Get vpn gateway
vpn_gateway = network_client.vpn_gateways.get(GROUP_NAME, VPN_GATEWAY)
print("Get vpn gateway:\n{}".format(vpn_gateway))
# Update vpn gateway
vpn_gateway = network_client.vpn_gateways.begin_update_tags(
GROUP_NAME, VPN_GATEWAY, {
"tags": {
"key1": "value1",
"key2": "value2"
}
}).result()
print("Update vpn gateway:\n{}".format(vpn_gateway))
# Delete vpn gateway
vpn_gateway = network_client.vpn_gateways.begin_delete(
GROUP_NAME, VPN_GATEWAY).result()
print("Delete vpn gateway.\n")
# Delete Group
resource_client.resource_groups.begin_delete(GROUP_NAME).result()
def get_client(self, subscription_id: str):
client = NetworkManagementClient(self.credentials.get_credentials('arm'),
subscription_id=subscription_id)
client._client.config.add_user_agent(get_user_agent())
return client
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
GROUP_NAME = "testgroupx"
VIRTUAL_MACHINE_SCALE_SET_NAME = "virtualmachinescaleset"
VMSS_EXTENSION_NAME = "vmssextensionxx"
NETWORK_NAME = "networknamex"
SUBNET_NAME = "subnetnamex"
your_password = '******' + ''.join(
random.choice(string.ascii_lowercase) for i in range(8))
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
network_client = NetworkManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
compute_client = ComputeManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
# Create resource group
resource_client.resource_groups.create_or_update(GROUP_NAME,
{"location": "eastus"})
# Create virtual network
network_client.virtual_networks.begin_create_or_update(
GROUP_NAME, NETWORK_NAME, {
'location': "eastus",
'address_space': {
'address_prefixes': ['10.0.0.0/16']
}
}).result()
subnet = network_client.subnets.begin_create_or_update(
GROUP_NAME, NETWORK_NAME, SUBNET_NAME, {
'address_prefix': '10.0.0.0/24'
}).result()
# Create virtual machine scale set
vmss = compute_client.virtual_machine_scale_sets.begin_create_or_update(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME, {
"sku": {
"tier": "Standard",
"capacity": "2",
"name": "Standard_D1_v2"
},
"location": "eastus",
"overprovision": True,
"virtual_machine_profile": {
"storage_profile": {
"image_reference": {
"offer": "UbuntuServer",
"publisher": "Canonical",
"sku": "18.04-LTS",
"version": "latest"
},
"os_disk": {
"caching": "ReadWrite",
"managed_disk": {
"storage_account_type": "Standard_LRS"
},
"create_option": "FromImage",
"disk_size_gb": "512"
},
},
"os_profile": {
"computer_name_prefix": "testPC",
"admin_username": "******",
"admin_password": your_password
},
"network_profile": {
"network_interface_configurations": [{
"name":
"testPC",
"primary":
True,
"enable_ipforwarding":
True,
"ip_configurations": [{
"name": "testPC",
"properties": {
"subnet": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID +
"/resourceGroups/" + GROUP_NAME +
"/providers/Microsoft.Network/virtualNetworks/"
+ NETWORK_NAME + "/subnets/" +
SUBNET_NAME + ""
}
}
}]
}]
}
},
"upgrade_policy": {
"mode": "Manual"
},
"upgrade_mode": "Manual"
}).result()
print("Create virtual machine scale set:\n{}".format(vmss))
# Create vmss extension
extension = compute_client.virtual_machine_scale_set_extensions.begin_create_or_update(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME, VMSS_EXTENSION_NAME, {
"location": "eastus",
"auto_upgrade_minor_version": True,
"publisher": "Microsoft.Azure.NetworkWatcher",
"type_properties_type": "NetworkWatcherAgentWindows",
"type_handler_version": "1.4",
}).result()
print("Create vmss extension:\n{}".format(extension))
# Get virtual machine scale set
vmss = compute_client.virtual_machine_scale_sets.get(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME)
print("Get virtual machine scale set:\n{}".format(vmss))
# Get vmss extesnion
extension = compute_client.virtual_machine_scale_set_extensions.get(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME, VMSS_EXTENSION_NAME)
print("Get vmss extension:\n{}".format(extension))
# Update virtual machine scale set
vmss = compute_client.virtual_machine_scale_sets.begin_update(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME, {
"sku": {
"tier": "Standard",
"capacity": "2",
"name": "Standard_D1_v2"
},
"upgrade_policy": {
"mode": "Manual"
}
}).result()
print("Update virtual machine scale set:\n{}".format(vmss))
# Update vmss extension
extension = compute_client.virtual_machine_scale_set_extensions.begin_update(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME, VMSS_EXTENSION_NAME, {
"auto_upgrade_minor_version": True,
}).result()
print("Update vmss extension:\n{}".format(extension))
# Delete vmss extension
compute_client.virtual_machine_scale_set_extensions.begin_delete(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME,
VMSS_EXTENSION_NAME).result()
print("Delete vmss extension.\n")
# Delete virtual machine scale set
compute_client.virtual_machine_scale_sets.begin_delete(
GROUP_NAME, VIRTUAL_MACHINE_SCALE_SET_NAME).result()
print("Delete virtual machine scale set.\n")
# Delete Group
resource_client.resource_groups.begin_delete(GROUP_NAME).result()
def __init__(self,
cred,
subs_id,
hub,
vmss_rg_name,
vmss_name,
storage,
pan_handle,
logger=None):
self.credentials = cred
self.subscription_id = subs_id
self.logger = logger
self.hub_name = hub
self.storage_name = storage
self.panorama_handler = pan_handle
self.vmss_table_name = re.sub(self.ALPHANUM, '',
vmss_name + 'vmsstable')
self.vmss_rg_name = vmss_rg_name
try:
self.resource_client = ResourceManagementClient(cred, subs_id)
self.compute_client = ComputeManagementClient(cred, subs_id)
self.network_client = NetworkManagementClient(cred, subs_id)
self.store_client = StorageManagementClient(cred, subs_id)
store_keys = self.store_client.storage_accounts.list_keys(
hub, storage).keys[0].value
self.table_service = TableService(account_name=storage,
account_key=store_keys)
except Exception as e:
self.logger.error("Getting Azure Infra handlers failed %s" %
str(e))
raise e
# Start -> List out all RGs and identify new spokes to mark them with tags.
# Look for Resource Groups (RGs) which do not have tags or does not have a
# a tag named "PANORAMA_PROGRAMMED".
# potential_new_spokes = [x.name for x in self.resource_client.resource_groups.list()\
# if not x.tags or not x.tags.get(self.RG_RULE_PROGRAMMED_TAG, None)]
# If the RG has a VMSS which has a tag named "PanoramaManaged" with a value
# as Hub Resource Group name then we know that this is a new spoke that is
# launched managed by the Hub and not yet programmed for NAT/Azure Instrumentation
# key.
# for rg in potential_new_spokes:
# fw_vm_list = [x for x in self.resource_client.resources.list_by_resource_group(rg)
# if x.type == self.VMSS_TYPE and self.filter_vmss(rg, x.name)]
# if fw_vm_list:
# rg_params = {'location': self.resource_client.resource_groups.get(rg).location}
# rg_params.update(tags={
# self.RG_RULE_PROGRAMMED_TAG : 'No',
# self.HUB_MANAGED_TAG : self.hub_name
# })
# self.resource_client.resource_groups.create_or_update(rg, rg_params)
# self.logger.info("RG %s marked as a spoke managed by this hub %s" % (rg, self.hub_name))
# End -> List out all RGs and identify new spokes to mark them with tags.
# Populate the list of spokes managed by this Azure hub.
rg_list = self.resource_client.resource_groups.list()
self.managed_spokes = []
self.managed_spokes.append(vmss_rg_name)
self.new_spokes = []
print(' virtual machine: {} created in resource group: {}'.format(virtualMachine.name, resourceGroup.name))
# init taskcluster clients
taskclusterQueueClient = taskcluster.Queue(taskcluster.optionsFromEnvironment())
# init azure clients
azureConfig = yaml.safe_load(open('{}/.azure.yaml'.format(os.getenv('HOME')), 'r'))
azureCredentials = ServicePrincipalCredentials(
client_id = azureConfig['client_id'],
secret = azureConfig['secret'],
tenant = azureConfig['tenant'])
azureComputeManagementClient = ComputeManagementClient(
azureCredentials,
azureConfig['subscription'])
azureNetworkManagementClient = NetworkManagementClient(
azureCredentials,
azureConfig['subscription'])
azureResourceManagementClient = ResourceManagementClient(
azureCredentials,
azureConfig['subscription'])
# provision until interrupted [ctrl + c]
try:
while True:
azureWorkerPools = yaml.safe_load(open('{}/azure-worker-pools.yaml'.format(os.path.dirname(__file__)), 'r'))
minions = getMinions()
for provisionerId in minions.keys():
for workerType in minions[provisionerId].keys():
pool = next(p for p in azureWorkerPools if p['name'] == '{}/{}'.format(provisionerId, workerType))
provision(provisionerId, workerType, minions[provisionerId][workerType], pool['regions'], pool['machine'], pool['capacity']['min'], pool['capacity']['max'])
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
GROUP_NAME = "testgroupx"
VIRTUAL_NETWORK_GATEWAY = "virtual_network_gatewayxxyyzz"
IP_CONFIGURATION_NAME = "ipconfigurationxx"
VIRTUAL_NETWORK_NAME = "virtualnetworkxx"
SUBNET = "GatewaySubnet" # Must be `GatewaySubnet`
PUBLIC_IP_ADDRESS_NAME = "publicipaddressxxxx"
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
network_client = NetworkManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
# Create resource group
resource_client.resource_groups.create_or_update(GROUP_NAME,
{"location": "eastus"})
# - init depended resources -
network_client.public_ip_addresses.begin_create_or_update(
GROUP_NAME, PUBLIC_IP_ADDRESS_NAME, {
'location': "eastus",
'public_ip_allocation_method': 'Dynamic',
'idle_timeout_in_minutes': 4
})
network_client.virtual_networks.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, {
"address_space": {
"address_prefixes": ["10.0.0.0/16"]
},
"location": "eastus"
}).result()
# Create subnet
network_client.subnets.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, SUBNET, {
"address_prefix": "10.0.0.0/24"
}).result()
# - end -
# Create virtual network gateway
virtual_network_gateway = network_client.virtual_network_gateways.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_GATEWAY, {
"ip_configurations": [{
"private_ip_allocation_method": "Dynamic",
"subnet": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME +
"/providers/Microsoft.Network/virtualNetworks/" +
VIRTUAL_NETWORK_NAME + "/subnets/" + SUBNET + ""
},
"public_ip_address": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME +
"/providers/Microsoft.Network/publicIPAddresses/" +
PUBLIC_IP_ADDRESS_NAME + ""
},
"name": IP_CONFIGURATION_NAME
}],
"gateway_type":
"Vpn",
"vpn_type":
"RouteBased",
"enable_bgp":
False,
"active_active":
False,
"enable_dns_forwarding":
False,
"sku": {
"name": "VpnGw1",
"tier": "VpnGw1"
},
"bgp_settings": {
"asn": "65515",
"bgp_peering_address": "10.0.1.30",
"peer_weight": "0"
},
"custom_routes": {
"address_prefixes": ["101.168.0.6/32"]
},
"location":
"eastus"
}).result()
print(
"Create virtual network gateway:\n{}".format(virtual_network_gateway))
# Get virtual network gateway
virtual_network_gateway = network_client.virtual_network_gateways.get(
GROUP_NAME, VIRTUAL_NETWORK_GATEWAY)
print("Get virtual network gateway:\n{}".format(virtual_network_gateway))
# Update virtual network gateway
virtual_network_gateway = network_client.virtual_network_gateways.begin_update_tags(
GROUP_NAME, VIRTUAL_NETWORK_GATEWAY, {
"tags": {
"tag1": "value1",
"tag2": "value2"
}
}).result()
print(
"Update virtual network gateway:\n{}".format(virtual_network_gateway))
# Delete virtual network gateway
virtual_network_gateway = network_client.virtual_network_gateways.begin_delete(
GROUP_NAME, VIRTUAL_NETWORK_GATEWAY).result()
print("Delete virtual network gateway.\n")
# Delete Group
resource_client.resource_groups.begin_delete(GROUP_NAME).result()
def _get_network_client(self, client_id, client_secret, tenant,
subscription):
credentials = ServicePrincipalCredentials(client_id=client_id,
secret=client_secret,
tenant=tenant)
return NetworkManagementClient(credentials, subscription)
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
GROUP_NAME = "testgroupx"
PRIVATE_ENDPOINT = "private_endpointxxyyzz"
SERVICE_NAME = PRIVATE_LINK_SERVICE = "private_link_servicexxyyzz"
LOAD_BALANCER_NAME = "loadbalancerxxx"
IP_CONFIGURATION_NAME = "ipconfigurationxxx"
VIRTUAL_NETWORK_NAME = "virtualnetworkxxx"
SUBNET = "subnetxxx"
SUBNET_2 = "subnetxxxz"
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
network_client = NetworkManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
# Create resource group
resource_client.resource_groups.create_or_update(GROUP_NAME,
{"location": "eastus"})
# - init depended resources -
# Create virtual network
network_client.virtual_networks.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, {
"address_space": {
"address_prefixes": ["10.0.0.0/16"]
},
"location": "eastus"
}).result()
# Create subnet
subnet = network_client.subnets.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, SUBNET, {
"address_prefix": "10.0.0.0/24",
'private_link_service_network_policies': 'disabled'
}).result()
# Create subnet
subnet = network_client.subnets.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, SUBNET_2, {
"address_prefix": "10.0.1.0/24",
'private_endpoint_network_policies': 'disabled'
}).result()
# Create load balancer
network_client.load_balancers.begin_create_or_update(
GROUP_NAME, LOAD_BALANCER_NAME, {
"location":
"eastus",
"sku": {
"name": "Standard"
},
"frontendIPConfigurations": [{
"name": IP_CONFIGURATION_NAME,
"subnet": {
"id": subnet.id
}
}]
}).result()
# Create private link service
network_client.private_link_services.begin_create_or_update(
GROUP_NAME, PRIVATE_LINK_SERVICE, {
"location":
"eastus",
"visibility": {
"subscriptions": [SUBSCRIPTION_ID]
},
"auto_approval": {
"subscriptions": [SUBSCRIPTION_ID]
},
"fqdns": ["fqdn1", "fqdn2", "fqdn3"],
"load_balancer_frontend_ip_configurations": [{
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/loadBalancers/" +
LOAD_BALANCER_NAME + "/frontendIPConfigurations/" +
IP_CONFIGURATION_NAME
}],
"ip_configurations": [{
"name": IP_CONFIGURATION_NAME,
"private_ip_address": "10.0.1.4",
"private_ipallocation_method": "Static",
"private_ip_address_version": "IPv4",
"subnet": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME +
"/providers/Microsoft.Network/virtualNetworks/" +
VIRTUAL_NETWORK_NAME + "/subnets/" + SUBNET
}
}]
}).result()
# - end -
# Create private endpoint
private_endpoint = network_client.private_endpoints.begin_create_or_update(
GROUP_NAME, PRIVATE_ENDPOINT, {
"location":
"eastus",
"private_link_service_connections": [{
"name":
SERVICE_NAME,
"private_link_service_id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME +
"/providers/Microsoft.Network/privateLinkServices/" +
SERVICE_NAME,
}],
"subnet": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/virtualNetworks/" +
VIRTUAL_NETWORK_NAME + "/subnets/" + SUBNET_2
}
}).result()
print("Create private endpoint:\n{}".format(private_endpoint))
# Get private endpoint
private_endpoint = network_client.private_endpoints.get(
GROUP_NAME, PRIVATE_ENDPOINT)
print("Get private endpoint:\n{}".format(private_endpoint))
# Delete private endpoint
private_endpoint = network_client.private_endpoints.begin_delete(
GROUP_NAME, PRIVATE_ENDPOINT).result()
print("Delete private endpoint.\n")
# Delete Group
resource_client.resource_groups.begin_delete(GROUP_NAME).result()
def getNetworkClient( spCred, subscription_id):
return NetworkManagementClient(
spCred,
subscription_id
)
__author__ = 'rafael'
# TODO: Install library
from azure.mgmt.compute import ComputeManagementClient, ComputeManagementClientConfiguration
from azure.mgmt.network import NetworkManagementClient, NetworkManagementClientConfiguration
# TODO: Replace this with your subscription id
subscription_id = '33333333-3333-3333-3333-333333333333'
# TODO: See above how to get a Credentials instance
credentials = ...
compute_client = ComputeManagementClient(
ComputeManagementClientConfiguration(credentials, subscription_id))
network_client = NetworkManagementClient(
NetworkManagementClientConfiguration(credentials, subscription_id))
import models.data.pgm_test_data as data
from msrestazure.azure_cloud import AZURE_US_GOV_CLOUD
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.resource import ResourceManagementClient
from azure.common.credentials import ServicePrincipalCredentials
from msrest.exceptions import AuthenticationError
# Create Service Principal
try:
credentials = ServicePrincipalCredentials(
client_id=data.CLIENT,
secret=data.KEY,
tenant=data.TENANT_ID,
cloud_environment=AZURE_US_GOV_CLOUD)
except AuthenticationError as e:
print(e)
network_client = NetworkManagementClient(
credentials,
data.subscription_id,
base_url=AZURE_US_GOV_CLOUD.endpoints.resource_manager)
resource_client = ResourceManagementClient(
credentials,
data.subscription_id,
base_url=AZURE_US_GOV_CLOUD.endpoints.resource_manager)
resource_client.providers.register('Microsoft.Network')
def run(job, **kwargs):
resource = kwargs.get('resource')
create_custom_fields_as_needed()
env_id = '{{ env_id }}'
env = Environment.objects.get(id=env_id)
rh = env.resource_handler.cast()
location = env.node_location
resource_group = '{{ resource_group }}'
virtual_net_name = '{{ virtual_net_name }}'
vpn_adress_prefix = '{{ vpn_adress_prefix }}'
subnet_adress_prefix = '{{ subnet_adress_prefix }}'
subnet_name = virtual_net_name + '_subnet'
set_progress("Connecting To Azure...")
credentials = ServicePrincipalCredentials(
client_id=rh.client_id,
secret=rh.secret,
tenant=rh.tenant_id,
)
network_client = NetworkManagementClient(credentials, rh.serviceaccount)
set_progress("Connection to Azure established")
#create the vpn
set_progress('Creating virtual network "%s"...' % virtual_net_name)
try:
async_vnet_creation = network_client.virtual_networks.create_or_update(
resource_group,
virtual_net_name,
{
'location': location,
'address_space': {
'address_prefixes': [vpn_adress_prefix]
}
}
)
async_vnet_creation.wait()
except CloudError as e:
set_progress('Azure Clouderror: {}'.format(e))
#create the subnet
set_progress('Creating subnet "%s"...' % subnet_name)
try:
async_subnet_creation = network_client.subnets.create_or_update(
resource_group,
virtual_net_name,
subnet_name,
{'address_prefix': subnet_adress_prefix}
)
subnet_info = async_subnet_creation.result()
except CloudError as e:
set_progress("Azure Clouderror: {}".format(e))
return "FAILURE", "Virtual network could not be created", e
assert subnet_info.name == subnet_name
set_progress('Subnet "%s" has been created.' % subnet_name)
resource.name = virtual_net_name + " - " + resource_group
resource.azure_virtual_net_name = virtual_net_name
resource.vpn_adress_prefix = vpn_adress_prefix
resource.resource_group_name = resource_group
resource.azure_location = location
resource.azure_rh_id = rh.id
resource.azure_subnet_name = subnet_name
resource.save()
return "SUCCESS", "The vpn and subnet have been successfully created", ""
def run_example():
"""Resource Group management example."""
#
# Create all clients with an Application (service principal) token provider
#
mystack_cloud = get_cloud_from_metadata_endpoint(
os.environ['ARM_ENDPOINT'])
# Set Storage Endpoint suffix
arm_url = mystack_cloud.endpoints.resource_manager
storage_endpoint_suffix = arm_url.replace(arm_url.split(".")[0],
"").strip('./')
subscription_id = os.environ.get(
'AZURE_SUBSCRIPTION_ID',
'11111111-1111-1111-1111-111111111111') # your Azure Subscription Id
credentials = ServicePrincipalCredentials(
client_id=os.environ['AZURE_CLIENT_ID'],
secret=os.environ['AZURE_CLIENT_SECRET'],
tenant=os.environ['AZURE_TENANT_ID'],
cloud_environment=mystack_cloud)
# By Default, use AzureStack supported profile
KnownProfiles.default.use(KnownProfiles.v2017_03_09_profile)
resource_client = ResourceManagementClient(
credentials,
subscription_id,
base_url=mystack_cloud.endpoints.resource_manager)
compute_client = ComputeManagementClient(
credentials,
subscription_id,
base_url=mystack_cloud.endpoints.resource_manager)
storage_client = StorageManagementClient(
credentials,
subscription_id,
base_url=mystack_cloud.endpoints.resource_manager)
network_client = NetworkManagementClient(
credentials,
subscription_id,
base_url=mystack_cloud.endpoints.resource_manager)
###########
# Prepare #
###########
# Create Resource group
print('\nCreate Resource Group')
resource_client.resource_groups.create_or_update(GROUP_NAME,
{'location': LOCATION})
# Create a storage account
print('\nCreate a storage account')
storage_async_operation = storage_client.storage_accounts.create(
GROUP_NAME, STORAGE_ACCOUNT_NAME, {
'sku': {
'name': 'standard_lrs'
},
'kind': 'storage',
'location': LOCATION
})
storage_async_operation.wait()
# Create a NIC
nic = create_nic(network_client)
#############
# VM Sample #
#############
# Create Linux VM
print('\nCreating Linux Virtual Machine')
vm_parameters = create_vm_parameters(nic.id, VM_REFERENCE['linux'],
storage_endpoint_suffix)
async_vm_creation = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, vm_parameters)
async_vm_creation.wait()
# Tag the VM
print('\nTag Virtual Machine')
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, {
'location': LOCATION,
'tags': {
'who-rocks': 'python',
'where': 'on azure'
}
})
async_vm_update.wait()
# Attach data disk
print('\nAttach Data Disk')
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, {
'location': LOCATION,
'storage_profile': {
'data_disks': [{
'name': 'mydatadisk1',
'disk_size_gb': 1,
'lun': 0,
'vhd': {
'uri':
"https://{}.blob.{}/vhds/mydatadisk1.vhd".format(
STORAGE_ACCOUNT_NAME, storage_endpoint_suffix)
},
'create_option': 'Empty'
}]
}
})
async_vm_update.wait()
# Get the virtual machine by name
print('\nGet Virtual Machine by Name')
virtual_machine = compute_client.virtual_machines.get(GROUP_NAME, VM_NAME)
# Detach data disk
print('\nDetach Data Disk')
data_disks = virtual_machine.storage_profile.data_disks
data_disks[:] = [disk for disk in data_disks if disk.name != 'mydatadisk1']
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, virtual_machine)
virtual_machine = async_vm_update.result()
# Deallocating the VM (resize prepare)
print('\nDeallocating the VM (resize prepare)')
async_vm_deallocate = compute_client.virtual_machines.deallocate(
GROUP_NAME, VM_NAME)
async_vm_deallocate.wait()
# Update OS disk size by 10Gb
print('\nUpdate OS disk size')
# Server is not returning the OS Disk size (None), possible bug in server
if not virtual_machine.storage_profile.os_disk.disk_size_gb:
print(
"\tServer is not returning the OS disk size, possible bug in the server?"
)
print("\tAssuming that the OS disk size is 256 GB")
virtual_machine.storage_profile.os_disk.disk_size_gb = 256
virtual_machine.storage_profile.os_disk.disk_size_gb += 10
async_vm_update = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, virtual_machine)
virtual_machine = async_vm_update.result()
# Start the VM
print('\nStart VM')
async_vm_start = compute_client.virtual_machines.start(GROUP_NAME, VM_NAME)
async_vm_start.wait()
# Restart the VM
print('\nRestart VM')
async_vm_restart = compute_client.virtual_machines.restart(
GROUP_NAME, VM_NAME)
async_vm_restart.wait()
# Stop the VM
print('\nStop VM')
async_vm_stop = compute_client.virtual_machines.power_off(
GROUP_NAME, VM_NAME)
async_vm_stop.wait()
# List VMs in subscription
print('\nList VMs in subscription')
for vm in compute_client.virtual_machines.list_all():
print("\tVM: {}".format(vm.name))
# List VM in resource group
print('\nList VMs in resource group')
for vm in compute_client.virtual_machines.list(GROUP_NAME):
print("\tVM: {}".format(vm.name))
# Delete VM
print('\nDelete VM')
async_vm_delete = compute_client.virtual_machines.delete(
GROUP_NAME, VM_NAME)
async_vm_delete.wait()
# Create Windows VM
print('\nCreating Windows Virtual Machine')
# Recycling NIC of previous VM
vm_parameters = create_vm_parameters(nic.id, VM_REFERENCE['windows'],
storage_endpoint_suffix)
async_vm_creation = compute_client.virtual_machines.create_or_update(
GROUP_NAME, VM_NAME, vm_parameters)
async_vm_creation.wait()
input("Press enter to delete this Resource Group.")
# Delete Resource group and everything in it
print('\nDelete Resource Group')
delete_async_operation = resource_client.resource_groups.delete(GROUP_NAME)
delete_async_operation.wait()
print("\nDeleted: {}".format(GROUP_NAME))
def __init__(
self,
location: str = None,
resource_group: str = None,
vnet: str = None,
security_group: str = None,
public_ingress: bool = None,
vm_size: str = None,
scheduler_vm_size: str = None,
vm_image: dict = {},
disk_size: int = None,
bootstrap: bool = None,
auto_shutdown: bool = None,
docker_image=None,
debug: bool = False,
marketplace_plan: dict = {},
**kwargs,
):
self.config = ClusterConfig(dask.config.get("cloudprovider.azure", {}))
self.scheduler_class = AzureVMScheduler
self.worker_class = AzureVMWorker
self.location = self.config.get("location", override_with=location)
if self.location is None:
raise ConfigError("You must configure a location")
self.resource_group = self.config.get("resource_group",
override_with=resource_group)
if self.resource_group is None:
raise ConfigError("You must configure a resource_group")
self.public_ingress = self.config.get("azurevm.public_ingress",
override_with=public_ingress)
self.subscription_id = get_cli_profile().get_subscription_id()
self.credentials = DefaultAzureCredential()
self.compute_client = ComputeManagementClient(self.credentials,
self.subscription_id)
self.network_client = NetworkManagementClient(self.credentials,
self.subscription_id)
self.vnet = self.config.get("azurevm.vnet", override_with=vnet)
if self.vnet is None:
raise ConfigError("You must configure a vnet")
self.security_group = self.config.get("azurevm.security_group",
override_with=security_group)
if self.security_group is None:
raise ConfigError(
"You must configure a security group which allows traffic on 8786 and 8787"
)
self.vm_size = self.config.get("azurevm.vm_size",
override_with=vm_size)
self.disk_size = self.config.get("azurevm.disk_size",
override_with=disk_size)
if self.disk_size > 1023:
raise ValueError(
"VM OS disk canot be larger than 1023. Please change the ``disk_size`` config option."
)
self.scheduler_vm_size = self.config.get(
"azurevm.scheduler_vm_size", override_with=scheduler_vm_size)
if self.scheduler_vm_size is None:
self.scheduler_vm_size = self.vm_size
self.gpu_instance = ("_NC" in self.vm_size.upper()
or "_ND" in self.vm_size.upper())
self.vm_image = self.config.get("azurevm.vm_image")
for key in vm_image:
self.vm_image[key] = vm_image[key]
self.bootstrap = self.config.get("azurevm.bootstrap",
override_with=bootstrap)
self.auto_shutdown = self.config.get("azurevm.auto_shutdown",
override_with=auto_shutdown)
self.debug = debug
self.marketplace_plan = marketplace_plan or self.config.get(
"azurevm.marketplace_plan")
if self.marketplace_plan:
# Check that self.marketplace_plan contains the right options with values
if not all(
self.marketplace_plan.get(item, "") != ""
for item in ["name", "publisher", "product"]):
raise ConfigError(
"""To create a virtual machine from Marketplace image or a custom image sourced
from a Marketplace image with a plan, all 3 fields 'name', 'publisher' and 'product' must be passed."""
)
self.options = {
"cluster": self,
"config": self.config,
"security_group": self.security_group,
"location": self.location,
"vm_image": self.vm_image,
"disk_size": self.disk_size,
"gpu_instance": self.gpu_instance,
"bootstrap": self.bootstrap,
"auto_shutdown": self.auto_shutdown,
"docker_image": self.docker_image,
"marketplace_plan": self.marketplace_plan,
}
self.scheduler_options = {
"vm_size": self.scheduler_vm_size,
"public_ingress": self.public_ingress,
**self.options,
}
self.worker_options = {"vm_size": self.vm_size, **self.options}
super().__init__(debug=debug, **kwargs)
def read(self):
"""Return an Azure cloud infrastructure configuration record.
Yields:
dict: An Azure cloud infrastructure configuration record.
"""
# pylint: disable=R0914
subscription_client = SubscriptionClient(self._credentials)
for i, sub in enumerate(subscription_client.subscriptions.list()):
subscription_id = str(sub.subscription_id)
_log.info('Found subscription #%d; subscription_id: %s; '
'display_name: %s',
i, subscription_id, sub.display_name)
# Initialize Azure clients for the current subscription.
creds = self._credentials
compute_client = ComputeManagementClient(creds, subscription_id)
network_client = NetworkManagementClient(creds, subscription_id)
storage_client = StorageManagementClient(creds, subscription_id)
resource_client = ResourceManagementClient(creds, subscription_id)
# Get iterators for each type of data.
vm_list = compute_client.virtual_machines.list_all()
app_gw_list = network_client.application_gateways.list_all()
lb_iter = network_client.load_balancers.list_all()
nic_list = network_client.network_interfaces.list_all()
nsg_list = network_client.network_security_groups.list_all()
pubip_list = network_client.public_ip_addresses.list_all()
storage_account_list = storage_client.storage_accounts.list()
resource_group_list = resource_client.resource_groups.list()
resource_list = resource_client.resources.list()
# Retrieve data using each iterator.
for record in itertools.chain(
_get_record(vm_list, 'virtual_machine',
subscription_id, self._max_recs),
_get_record(app_gw_list, 'app_gateway',
subscription_id, self._max_recs),
_get_record(lb_iter, 'lb',
subscription_id, self._max_recs),
_get_record(nic_list, 'nic',
subscription_id, self._max_recs),
_get_record(nsg_list, 'nsg',
subscription_id, self._max_recs),
_get_record(pubip_list, 'public_ip',
subscription_id, self._max_recs),
_get_record(storage_account_list, 'storage_account',
subscription_id, self._max_recs),
_get_record(resource_group_list, 'resource_group',
subscription_id, self._max_recs),
_get_record(resource_list, 'resource',
subscription_id, self._max_recs),
):
yield record
# Break after pulling data for self._max_subs number of
# subscriptions. Note that if self._max_subs is 0 or less,
# then the following condition never evaluates to True.
if i + 1 == self._max_subs:
_log.info('Ending subscriptions fetch due to '
'_max_subs: %d', self._max_subs)
break
def get_network_client() -> NetworkManagementClient:
return NetworkManagementClient(get_identity(), get_subscription())
def main():
validate_env()
location = os.getenv('AZURE_LOCATION', 'East US')
credentials = azure.common.credentials.ServicePrincipalCredentials(
client_id=os.environ['AZURE_CLIENT_ID'],
secret=os.environ['AZURE_CLIENT_SECRET'],
tenant=os.environ['AZURE_TENANT_ID'])
subscription_id = os.environ['AZURE_SUBSCRIPTION_ID']
template = TemplateLink(uri=os.environ['AZURE_TEMPLATE_URL'])
# tenant_id = os.environ.get('AZURE_TENANT_ID')
# client_id = os.environ.get('AZURE_CLIENT_ID')
# client_secret = os.environ.get('AZURE_CLIENT_SECRET')
group_name = 'testing' + ''.join(random.choice('01234567890abcdef') for n in range(10))
deployment_name = 'deployment{}'.format(uuid.uuid4().hex)
rmc = ResourceManagementClient(credentials, subscription_id)
template_parameters = get_env_params()
# Output resource group
print("Resource group name: {}".format(group_name))
print("Deployment name: {}".format(deployment_name))
azure_cluster = {
'resource_group_name': group_name,
'deployment_name': deployment_name}
pkgpanda.util.write_json('azure-cluster.json', azure_cluster)
# Create a new resource group
print("Creating new resource group in location: {}".format(location))
if rmc.resource_groups.check_existence(group_name):
print("ERROR: Group name already exists / taken: {}".format(group_name))
rmc.resource_groups.create_or_update(
group_name,
ResourceGroup(location=location))
test_successful = False
try:
deployment_properties = DeploymentProperties(
template_link=template,
mode=DeploymentMode.incremental,
parameters=template_parameters)
# Use RPC against azure to validate the ARM template is well-formed
result = rmc.deployments.validate(group_name, deployment_name, properties=deployment_properties)
if result.error:
print("Template verification failed\n{}".format(result.error), file=sys.stderr)
sys.exit(1)
# Actually create a template deployment
print("Creating template deployment ...")
deploy_poller = rmc.deployments.create_or_update(group_name, deployment_name, deployment_properties)
# Stop after 45 attempts (each one takes up to one minute)
@retry(stop_max_attempt_number=45)
def poll_deploy():
res = deploy_poller.result(timeout=60)
print("Current deploy state: {}".format(res.properties.provisioning_state))
assert deploy_poller.done(), "Not done deploying."
print("Waiting for template to deploy ...")
try:
poll_deploy()
except:
print("Current deploy status:\n{}".format(deploy_poller.result(0)))
raise
print("Template deployed successfully")
assert deploy_poller.done(), "Deployment failed / polling didn't reach deployment done."
deployment_result = deploy_poller.result()
print(deployment_result.properties.outputs)
master_lb = deployment_result.properties.outputs['masterFQDN']['value']
print("Template deployed using SSH private key: https://mesosphere.onelogin.com/notes/18444")
print("For troubleshooting, master0 can be reached using: ssh -p 2200 {}@{}".format(
get_value('linuxAdminUsername'), master_lb))
# Run test now, so grab IPs
nmc = NetworkManagementClient(credentials, subscription_id)
ip_buckets = {
'master': [],
'private': [],
'public': []}
for resource in rmc.resource_groups.list_resources(
group_name, filter=("resourceType eq 'Microsoft.Network/networkInterfaces' or "
"resourceType eq 'Microsoft.Compute/virtualMachineScaleSets'")):
if resource.type == 'Microsoft.Network/networkInterfaces':
nics = [nmc.network_interfaces.get(group_name, resource.name)]
elif resource.type == 'Microsoft.Compute/virtualMachineScaleSets':
nics = list(nmc.network_interfaces.list_virtual_machine_scale_set_network_interfaces(
virtual_machine_scale_set_name=resource.name, resource_group_name=group_name))
else:
raise('Unexpected resourceType: {}'.format(resource.type))
for bucket_name in ip_buckets.keys():
if bucket_name in resource.name:
for n in nics:
for config in n.ip_configurations:
ip_buckets[bucket_name].append(config.private_ip_address)
print('Detected IP configuration: {}'.format(ip_buckets))
with SSHTunnel(get_value('linuxAdminUsername'), 'ssh_key', master_lb, port=2200) as t:
integration_test(
tunnel=t,
test_dir='/home/{}'.format(get_value('linuxAdminUsername')),
dcos_dns=ip_buckets['master'][0],
master_list=ip_buckets['master'],
agent_list=ip_buckets['private'],
public_agent_list=ip_buckets['public'],
provider='azure',
test_dns_search=False,
add_env=get_test_config(),
pytest_cmd=os.getenv('DCOS_PYTEST_CMD', "py.test -rs -vv -m 'not ccm' ") + os.getenv('CI_FLAGS', ''))
test_successful = True
except Exception as ex:
traceback.print_exc()
print("ERROR: exception {}".format(ex))
raise
finally:
if os.getenv('AZURE_CLEANUP') == 'false':
print("Cluster must be cleaned up manually")
print("Cluster details: {}".format(azure_cluster))
else:
# Send a delete request
# TODO(cmaloney): The old code had a retry around this:
# @retry(wait_exponential_multiplier=1000, wait_exponential_max=60*1000, stop_max_delay=(30*60*1000))
poller = rmc.resource_groups.delete(group_name)
# poll for the delete to complete
print("Deleting resource group: {} ...".format(group_name))
@retry(wait_fixed=(5 * 1000), stop_max_delay=(60 * 60 * 1000))
def wait_for_delete():
assert poller.done(), "Timed out waiting for delete"
print("Waiting for delete ...")
wait_for_delete()
print("Clean up successful")
if test_successful:
print("Azure test deployment succeeded")
else:
print("ERROR: Azure test deployment failed", file=sys.stderr)
sys.exit(2)
def main():
SUBSCRIPTION_ID = os.environ.get("SUBSCRIPTION_ID", None)
GROUP_NAME = "testgroupx"
VIRTUAL_NETWORK_PEERING = "virtual_network_peeringxxyyzz"
VIRTUAL_NETWORK_NAME = "virtualnetwork"
SUBNET = "subnetxxx"
REMOTE_NETWORK_NAME = "remotenetworkxxx"
REMOTE_SUBNET = "remotesubnetxxx"
# Create client
# For other authentication approaches, please see: https://pypi.org/project/azure-identity/
resource_client = ResourceManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
network_client = NetworkManagementClient(
credential=DefaultAzureCredential(), subscription_id=SUBSCRIPTION_ID)
# Create resource group
resource_client.resource_groups.create_or_update(GROUP_NAME,
{"location": "eastus"})
# - init depended resources -
# Create virtual network
network_client.virtual_networks.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, {
"address_space": {
"address_prefixes": ["10.0.0.0/16"]
},
"location": "eastus"
}).result()
# Create subnet
network_client.subnets.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, SUBNET, {
"address_prefix": "10.0.0.0/24"
}).result()
# Create remote network
network_client.virtual_networks.begin_create_or_update(
GROUP_NAME, REMOTE_NETWORK_NAME, {
"address_space": {
"address_prefixes": ["10.2.0.0/16"]
},
"location": "eastus"
}).result()
# Create subnet
network_client.subnets.begin_create_or_update(
GROUP_NAME, REMOTE_NETWORK_NAME, REMOTE_SUBNET, {
"address_prefix": "10.2.0.0/24"
}).result()
# - end -
# Create virtual network peering
virtual_network_peering = network_client.virtual_network_peerings.begin_create_or_update(
GROUP_NAME, VIRTUAL_NETWORK_NAME, VIRTUAL_NETWORK_PEERING, {
"allow_virtual_network_access": True,
"allow_forwarded_traffic": True,
"allow_gateway_transit": False,
"use_remote_gateways": False,
"remote_virtual_network": {
"id":
"/subscriptions/" + SUBSCRIPTION_ID + "/resourceGroups/" +
GROUP_NAME + "/providers/Microsoft.Network/virtualNetworks/" +
REMOTE_NETWORK_NAME + ""
}
}).result()
print(
"Create virtual network peering:\n{}".format(virtual_network_peering))
# Get virtual network peering
virtual_network_peering = network_client.virtual_network_peerings.get(
GROUP_NAME, VIRTUAL_NETWORK_NAME, VIRTUAL_NETWORK_PEERING)
print("Get virtual network peering:\n{}".format(virtual_network_peering))
# Delete virtual network peering
virtual_network_peering = network_client.virtual_network_peerings.begin_delete(
GROUP_NAME, VIRTUAL_NETWORK_NAME, VIRTUAL_NETWORK_PEERING).result()
print("Delete virtual network peering.\n")
# Delete Group
resource_client.resource_groups.begin_delete(GROUP_NAME).result()
def network_client(self):
return NetworkManagementClient(self.credentials, self.subscription_id)
def run(self, Subcription_id, Group_Name, Location, VM_Name, Client_Id, Secret, Tenant_Id):
#Defining Variables for VM creation
SUBSCRIPTION_ID = Subcription_id
GROUP_NAME = Group_Name
LOCATION = Location
VM_NAME = VM_Name
#Function for Creation Availability Set for Virtual Machine
def create_availability_set(compute_client):
avset_params = {
'location': LOCATION,
'sku': {'name': 'Aligned'},
'platform_fault_domain_count': 2
}
availability_set_result = compute_client.availability_sets.create_or_update(
GROUP_NAME,
'myAVSet',
avset_params
)
#Function for Creation Group for Virtual Machine
def create_resource_group(resource_group_client):
resource_group_params = {'location': LOCATION}
resource_group_result = resource_group_client.resource_groups.create_or_update(
GROUP_NAME,
resource_group_params
)
#Function to get credentials for Authentication
def get_credentials():
credentials = ServicePrincipalCredentials(
client_id=Client_Id,
secret=Secret,
tenant=Tenant_Id
)
return credentials
#Function To create Public IP address
def create_public_ip_address(network_client):
public_ip_addess_params = {
'location': LOCATION,
'public_ip_allocation_method': 'Dynamic'
}
creation_result = network_client.public_ip_addresses.create_or_update(
GROUP_NAME,
'myIPAddress',
public_ip_addess_params
)
return creation_result.result()
#Function to create Vnet for virtual machine
def create_vnet(network_client):
vnet_params = {
'location': LOCATION,
'address_space': {
'address_prefixes': ['10.0.0.0/16']
}
}
creation_result = network_client.virtual_networks.create_or_update(
GROUP_NAME,
'myVNet',
vnet_params
)
return creation_result.result()
#Function to create Subnet for virtual machine
def create_subnet(network_client):
subnet_params = {
'address_prefix': '10.0.0.0/24'
}
creation_result = network_client.subnets.create_or_update(
GROUP_NAME,
'myVNet',
'mySubnet',
subnet_params
)
return creation_result.result()
#Function to create nic for virtual machine
def create_nic(network_client):
subnet_info = network_client.subnets.get(
GROUP_NAME,
'myVNet',
'mySubnet'
)
publicIPAddress = network_client.public_ip_addresses.get(
GROUP_NAME,
'myIPAddress'
)
nic_params = {
'location': LOCATION,
'ip_configurations': [{
'name': 'myIPConfig',
'public_ip_address': publicIPAddress,
'subnet': {
'id': subnet_info.id
}
}]
}
creation_result = network_client.network_interfaces.create_or_update(
GROUP_NAME,
'myNic',
nic_params
)
return creation_result.result()
#Function to create the virtual machine using above resources
def create_vm(network_client, compute_client):
nic = network_client.network_interfaces.get(
GROUP_NAME,
'myNic'
)
avset = compute_client.availability_sets.get(
GROUP_NAME,
'myAVSet'
)
vm_parameters = {
'location': LOCATION,
'os_profile': {
'computer_name': VM_NAME,
'admin_username': '******',
'admin_password': '******'
},
'hardware_profile': {
'vm_size': 'Standard_A0'
},
'storage_profile': {
'image_reference': {
'publisher': 'MicrosoftWindowsServer',
'offer': 'WindowsServer',
'sku': '2012-R2-Datacenter',
'version': 'latest'
}
},
'network_profile': {
'network_interfaces': [{
'id': nic.id
}]
},
'availability_set': {
'id': avset.id
}
}
creation_result = compute_client.virtual_machines.create_or_update(
GROUP_NAME,
VM_NAME,
vm_parameters
)
return creation_result.result()
credentials = get_credentials()
#Defining Management Clients for creating and managing resources for azure
resource_group_client = ResourceManagementClient(
credentials,
SUBSCRIPTION_ID
)
network_client = NetworkManagementClient(
credentials,
SUBSCRIPTION_ID
)
compute_client = ComputeManagementClient(
credentials,
SUBSCRIPTION_ID
)
#Calling all above defined function to create resources and finally Virtual Machine
create_resource_group(resource_group_client)
create_availability_set(compute_client)
creation_result = create_public_ip_address(network_client)
creation_result = create_vnet(network_client)
creation_result = create_subnet(network_client)
creation_result = create_nic(network_client)
creation_result = create_vm(network_client, compute_client)
print("VM created Successfully")
def network(self) -> NetworkManagementClient:
return NetworkManagementClient(credential=self.credential,
subscription_id=self.subscription_id)
def __init__(
self,
location: str = None,
resource_group: str = None,
vnet: str = None,
security_group: str = None,
public_ingress: bool = None,
vm_size: str = None,
scheduler_vm_size: str = None,
vm_image: dict = {},
bootstrap: bool = None,
auto_shutdown: bool = None,
docker_image=None,
**kwargs,
):
self.config = dask.config.get("cloudprovider.azure.azurevm", {})
self.scheduler_class = AzureVMScheduler
self.worker_class = AzureVMWorker
self.location = (location if location is not None else
dask.config.get("cloudprovider.azure.location"))
if self.location is None:
raise ConfigError("You must configure a location")
self.resource_group = (resource_group if resource_group is not None
else self.config.get("resource_group"))
if self.resource_group is None:
raise ConfigError("You must configure a resource_group")
self.public_ingress = (public_ingress if public_ingress is not None
else self.config.get("public_ingress"))
self.credentials, self.subscription_id = get_azure_cli_credentials()
self.compute_client = ComputeManagementClient(self.credentials,
self.subscription_id)
self.network_client = NetworkManagementClient(self.credentials,
self.subscription_id)
self.vnet = vnet if vnet is not None else self.config.get("vnet")
if self.vnet is None:
raise ConfigError("You must configure a vnet")
self.security_group = (security_group if security_group is not None
else self.config.get("security_group"))
if self.security_group is None:
raise ConfigError(
"You must configure a security group which allows traffic on 8786 and 8787"
)
self.vm_size = vm_size if vm_size is not None else self.config.get(
"vm_size")
self.scheduler_vm_size = (scheduler_vm_size
if scheduler_vm_size is not None else
self.config.get("scheduler_vm_size"))
if self.scheduler_vm_size is None:
self.scheduler_vm_size = self.vm_size
self.gpu_instance = "NC" in self.vm_size or "ND" in self.vm_size
self.vm_image = self.config.get("vm_image")
for key in vm_image:
self.vm_image[key] = vm_image[key]
self.bootstrap = (bootstrap if bootstrap is not None else
self.config.get("bootstrap"))
self.auto_shutdown = (auto_shutdown if auto_shutdown is not None else
self.config.get("auto_shutdown"))
self.docker_image = docker_image or self.config.get("docker_image")
self.options = {
"cluster": self,
"config": self.config,
"security_group": self.security_group,
"location": self.location,
"vm_image": self.vm_image,
"gpu_instance": self.gpu_instance,
"bootstrap": self.bootstrap,
"auto_shutdown": self.auto_shutdown,
"docker_image": self.docker_image,
}
self.scheduler_options = {
"vm_size": self.scheduler_vm_size,
"public_ingress": self.public_ingress,
**self.options,
}
self.worker_options = {"vm_size": self.vm_size, **self.options}
super().__init__(**kwargs)
#--------------------------------------------------------------------------
# variables
#--------------------------------------------------------------------------
AZURE_LOCATION = 'eastus'
RESOURCE_GROUP = "myResourceGroup"
VIRTUAL_NETWORK_NAME = "myVirtualNetwork"
SERVICE_ENDPOINT_POLICY_NAME = "myServiceEndpointPolicy"
#--------------------------------------------------------------------------
# management clients
#--------------------------------------------------------------------------
credentials = ServicePrincipalCredentials(client_id=CLIENT_ID,
secret=CLIENT_SECRET,
tenant=TENANT_ID)
mgmt_client = NetworkManagementClient(credentials, SUBSCRIPTION_ID)
resource_client = ResourceManagementClient(credentials, SUBSCRIPTION_ID)
#--------------------------------------------------------------------------
# resource group (prerequisite)
#--------------------------------------------------------------------------
print("Creating Resource Group")
resource_client.resource_groups.create_or_update(
resource_group_name=RESOURCE_GROUP,
parameters={'location': AZURE_LOCATION})
#--------------------------------------------------------------------------
# /VirtualNetworks/put/Create virtual network with service endpoints and service endpoint policy[put]
#--------------------------------------------------------------------------
print(
"Create virtual network with service endpoints and service endpoint policy"
},
'bgp_settings': {
'asn': 65515,
'peer_weight': 0
},
'virtual_hub': {
'id':
"/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualHubs/%s"
% (os.environ.get('ARM_SUBSCRIPTION_ID'), os.environ.get('GROUP_NAME'),
os.environ.get('VHUB_NAME'))
},
'vpn_gateway_scale_unit': 1
}
# Connect to Azure APIs and get session details
network_client = NetworkManagementClient(credentials,
os.environ.get('ARM_SUBSCRIPTION_ID'))
# Delay for 30 seconds before deleting resources
time.sleep(60)
# Create VPNG
async_vpng_creation = network_client.vpn_gateways.create_or_update(
os.environ.get('GROUP_NAME'),
os.environ.get('VPNG_NAME'),
VPNG_PARAMS,
custom_headers=None,
raw=False,
polling=True)
async_vpng_creation.wait()
print(async_vpng_creation.result())
def main():
parser = argparse.ArgumentParser(
description=
'This program generates SSH config from Azure ARM VM inventry in subscription'
)
parser.add_argument('--version',
action='version',
version=_AZURE_SSH_CONFIG_VERSION)
parser.add_argument(
'--init',
action='store_true',
help=
'Create template client profile at $HOME/.azure/azuresshconfig.json only if there is no existing one'
)
parser.add_argument(
'--profile',
help=
'Specify azure client profile file to use ($HOME/.azure/azuresshconfig.json by default)'
)
parser.add_argument(
'--output',
help=
'Specify ssh config file path ($HOME/.ssh/config by default). Or specify "stdout" if you want to print its output to STDOUT'
)
parser.add_argument('--user', help='SSH username to use for all hosts')
parser.add_argument('--identityfile',
help='SSH identity file to use for all hosts')
parser.add_argument(
'--private',
action='store_true',
help='Use private IP addresses (Public IP is used by default)')
parser.add_argument(
'--resourcegroups',
help=
'A comma-separated list of resource group to be considered for ssh-config generation (all resource groups by default)'
)
parser.add_argument(
'--params',
help=
'Any ssh-config params you want to add with query-string format: key1=value1&key2=value2&...'
)
args = parser.parse_args()
if args.init:
# check if $HOME/.azure directory exists and create if not exists
azure_config_home = '{}/.azure'.format(os.environ['HOME'])
if not os.path.exists(azure_config_home):
os.makedirs(azure_config_home)
# Initialize azure client profile file
if not os.path.exists(_DEFAULT_AZURE_SSH_CONFIG_JSON_FILE):
ClientProfileConfig.generate_template(
_DEFAULT_AZURE_SSH_CONFIG_JSON_FILE)
print("Created template client profile!: {}".format(
_DEFAULT_AZURE_SSH_CONFIG_JSON_FILE))
else:
print_err(
"No action has done since client profile file already exist!: {}"
.format(_DEFAULT_AZURE_SSH_CONFIG_JSON_FILE))
quit()
### Args Validation
client_profile_file = args.profile if args.profile else _DEFAULT_AZURE_SSH_CONFIG_JSON_FILE
if not os.path.exists(client_profile_file):
print_err("Client profile doesn't exist: {0}\n"
"For the client profile detail, refer to {1}".format(
client_profile_file, _AZURE_SSH_CONFIG_HOME_SITE))
quit()
ssh_config_output = args.output if args.output else _DEFAULT_SSH_CONFIG_FILE
ssh_default_user = args.user if args.user else ''
ssh_default_identityfile = args.identityfile if args.identityfile else ''
option_private_ip = args.private
filter_resource_groups = []
if args.resourcegroups:
lower_rg = args.resourcegroups.lower()
rslist = lower_rg.split(',')
if len(rslist) > 0:
filter_resource_groups = rslist
additional_params = []
if args.params:
pairlist = args.params.split('&')
for s in pairlist:
p = s.split('=')
if (len(p) == 2):
additional_params.append(p)
### Load Config
cconf = ClientProfileConfig(client_profile_file)
### Get Target virtual machines info List
credentials = ServicePrincipalCredentials(cconf.client_id,
cconf.client_scret,
tenant=cconf.tenant_id)
compute_client = ComputeManagementClient(credentials,
cconf.subscription_id)
network_client = NetworkManagementClient(credentials,
cconf.subscription_id)
target_vm_list = []
for vm in compute_client.virtual_machines.list_all():
target_vm = {}
target_vm['name'] = vm.name
vm_rgroup = get_resorucegroup_from_vmid(vm.id)
# Filtering by resource group if needed
if len(filter_resource_groups) > 0:
r = vm_rgroup.lower()
if not (r in filter_resource_groups):
continue # skip
network_interfaces = vm.network_profile.network_interfaces
for ni in network_interfaces:
ni_info = get_network_interface_info(network_client, ni.id)
if option_private_ip:
if exists_in_dict('private_ip', ni_info):
target_vm['access_ip'] = ni_info['private_ip']
else:
if exists_in_dict('public_ip', ni_info):
target_vm['access_ip'] = ni_info['public_ip']
if exists_in_dict('access_ip', target_vm):
break
# Add only vm that has access_ip
if exists_in_dict('access_ip', target_vm):
target_vm_list.append(target_vm)
### Generate and append config block to ssh-config file
scblock = SSHConfigBlock()
for v in target_vm_list:
params = {}
if ssh_default_user:
params['User'] = ssh_default_user
if ssh_default_identityfile:
params['IdentityFile'] = ssh_default_identityfile
if len(additional_params) > 0:
for pset in additional_params:
params[pset[0]] = pset[1]
scblock.add_entry(v['name'], v['access_ip'], params)
ssh_config_block = scblock.to_string()
if ssh_config_output.lower() == 'stdout':
print("{}".format(ssh_config_block))
else:
ssh_config = SSHConfig(ssh_config_output)
if ssh_config.block_exists():
ssh_config.update_block(ssh_config_block)
else:
ssh_config.append_block(ssh_config_block)
print("Done! Updated: {}".format(ssh_config.sshconfig))
def network_models(self):
self.log("Getting network models...")
return NetworkManagementClient.models("2017-11-01")
