def create_monitoring_resource(auth_client, resource_client, compute_client, network_client, blob_client, table_client, config, resources_path, bootstrap_file, monitoring_files): # type: (azure.mgmt.authorization.AuthorizationManagementClient, # azure.mgmt.resource.resources.ResourceManagementClient, # azure.mgmt.compute.ComputeManagementClient, # azure.mgmt.network.NetworkManagementClient, # azure.storage.blob.BlockBlobService, # azure.cosmosdb.table.TableService, # dict, str, pathlib.Path, Tuple[str, pathlib.Path], # List[Tuple[str, pathlib.Path]]) -> None """Create a monitoring resource :param azure.mgmt.authorization.AuthorizationManagementClient auth_client: auth client :param azure.mgmt.resource.resources.ResourceManagementClient resource_client: resource client :param azure.mgmt.compute.ComputeManagementClient compute_client: compute client :param azure.mgmt.network.NetworkManagementClient network_client: network client :param azure.storage.blob.BlockBlobService blob_client: blob client :param azure.cosmosdb.table.TableService table_client: table client :param dict config: configuration dict :param pathlib.Path: resources path :param Tuple[str, pathlib.Path] bootstrap_file: customscript bootstrap file :param List[Tuple[str, pathlib.Path]] monitoring_files: configurable monitoring files """ ms = settings.monitoring_settings(config) # get subscription id for msi sub_id = settings.credentials_management(config).subscription_id if util.is_none_or_empty(sub_id): raise ValueError('Management subscription id not specified') # check if cluster already exists logger.debug('checking if monitoring resource exists') try: vm = compute_client.virtual_machines.get( resource_group_name=ms.resource_group, vm_name=settings.generate_virtual_machine_name(ms, 0)) raise RuntimeError( 'Existing virtual machine {} found for monitoring'.format(vm.id)) except msrestazure.azure_exceptions.CloudError as e: if e.status_code == 404: pass else: raise # confirm before proceeding if not util.confirm_action(config, 'create monitoring resource'): return # create resource group if it doesn't exist resource.create_resource_group(resource_client, ms.resource_group, ms.location) # check for conflicting options servconf = settings.monitoring_services_settings(config) if servconf.lets_encrypt_enabled and not ms.public_ip.enabled: raise ValueError( 'cannot create a monitoring resource without a public ip and ' 'lets encrypt enabled') # create storage container storage.create_storage_containers_nonbatch(blob_client, table_client, None, 'monitoring') # configure yaml files and write to resources if servconf.lets_encrypt_enabled and ms.public_ip.enabled: with monitoring_files['compose'][1].open('r') as f: compdata = f.read() else: with monitoring_files['compose-nonginx'][1].open('r') as f: compdata = f.read() with monitoring_files['prometheus'][1].open('r') as f: promdata = f.read() with monitoring_files['nginx'][1].open('r') as f: nginxdata = f.read() compdata = compdata.replace('{GRAFANA_ADMIN_USER}', servconf.grafana.admin_user).replace( '{GRAFANA_ADMIN_PASSWORD}', servconf.grafana.admin_password) if servconf.prometheus.port is not None: if servconf.lets_encrypt_enabled and ms.public_ip.enabled: compdata = compdata.replace( '{PROMETHEUS_PORT}', '- "{p}:{p}"'.format(p=servconf.prometheus.port)) nginxdata = nginxdata.replace('{PROMETHEUS_PORT}', servconf.prometheus.port) else: compdata = compdata.replace('{PROMETHEUS_PORT}', servconf.prometheus.port) else: if servconf.lets_encrypt_enabled and ms.public_ip.enabled: compdata = compdata.replace('{PROMETHEUS_PORT}', '') nginxdata = nginxdata.replace('{PROMETHEUS_PORT}', '9090') else: compdata = compdata.replace('{PROMETHEUS_PORT}', '9090') promdata = promdata.replace('{PROMETHEUS_SCRAPE_INTERVAL}', servconf.prometheus.scrape_interval) compyml = resources_path / monitoring_files['compose'][0] promyml = resources_path / monitoring_files['prometheus'][0] nginxconf = resources_path / monitoring_files['nginx'][0] with compyml.open('wt') as f: f.write(compdata) with promyml.open('wt') as f: f.write(promdata) with nginxconf.open('wt') as f: f.write(nginxdata) del compdata del promdata del nginxdata monitoring_files = [ bootstrap_file, monitoring_files['dashboard'], (monitoring_files['compose'][0], compyml), (monitoring_files['prometheus'][0], promyml), (monitoring_files['nginx'][0], nginxconf), ] add_dash = None if util.is_not_empty(servconf.grafana.additional_dashboards): add_dash = resources_path / 'additional_dashboards.txt' with add_dash.open('wt') as f: for key in servconf.grafana.additional_dashboards: f.write('{},{}\n'.format( key, servconf.grafana.additional_dashboards[key])) monitoring_files.append((add_dash.name, add_dash)) # upload scripts to blob storage for customscript vm extension blob_urls = storage.upload_for_nonbatch(blob_client, monitoring_files, 'monitoring') try: compyml.unlink() except OSError: pass try: promyml.unlink() except OSError: pass try: nginxconf.unlink() except OSError: pass if add_dash is not None: try: add_dash.unlink() except OSError: pass # async operation dictionary async_ops = {} # create nsg async_ops['nsg'] = resource.AsyncOperation( functools.partial(resource.create_network_security_group, network_client, ms)) # use dynamic ips for private private_ips = None logger.debug('using dynamic private ip address allocation') # create virtual network and subnet if specified vnet, subnet = resource.create_virtual_network_and_subnet( resource_client, network_client, ms.virtual_network.resource_group, ms.location, ms.virtual_network) # create public ips pips = None if ms.public_ip.enabled: async_ops['pips'] = {} async_ops['pips'][0] = resource.AsyncOperation( functools.partial(resource.create_public_ip, network_client, ms, 0)) logger.debug('waiting for public ips to provision') pips = {} for offset in async_ops['pips']: pip = async_ops['pips'][offset].result() logger.info(('public ip: {} [provisioning_state={} ip_address={} ' 'public_ip_allocation={}]').format( pip.id, pip.provisioning_state, pip.ip_address, pip.public_ip_allocation_method)) pips[offset] = pip else: logger.info('public ip is disabled for monitoring resource') # get nsg logger.debug('waiting for network security group to provision') nsg = async_ops['nsg'].result() # create nics async_ops['nics'] = {} async_ops['nics'][0] = resource.AsyncOperation( functools.partial(resource.create_network_interface, network_client, ms, subnet, nsg, private_ips, pips, 0)) # wait for nics to be created logger.debug('waiting for network interfaces to provision') nics = {} for offset in async_ops['nics']: nic = async_ops['nics'][offset].result() logger.info( ('network interface: {} [provisioning_state={} private_ip={} ' 'private_ip_allocation_method={} network_security_group={} ' 'accelerated_networking={}]').format( nic.id, nic.provisioning_state, nic.ip_configurations[0].private_ip_address, nic.ip_configurations[0].private_ip_allocation_method, nsg.name if nsg is not None else None, nic.enable_accelerated_networking)) nics[offset] = nic # read or generate ssh keys if util.is_not_empty(ms.ssh.ssh_public_key_data): key_data = ms.ssh.ssh_public_key_data else: # create universal ssh key for all vms if not specified ssh_pub_key = ms.ssh.ssh_public_key if ssh_pub_key is None: _, ssh_pub_key = crypto.generate_ssh_keypair( ms.ssh.generated_file_export_path, crypto.get_monitoring_ssh_key_prefix()) # read public key data with ssh_pub_key.open('rb') as fd: key_data = fd.read().decode('utf8') ssh_pub_key = compute_client.virtual_machines.models.SshPublicKey( path='/home/{}/.ssh/authorized_keys'.format(ms.ssh.username), key_data=key_data, ) # create vms async_ops['vms'] = {} async_ops['vms'][0] = resource.AsyncOperation( functools.partial(resource.create_virtual_machine, compute_client, ms, None, nics, None, ssh_pub_key, 0, enable_msi=True)) # wait for vms to be created logger.info('waiting for {} virtual machines to provision'.format( len(async_ops['vms']))) vms = {} for offset in async_ops['vms']: vms[offset] = async_ops['vms'][offset].result() logger.debug('{} virtual machines created'.format(len(vms))) # create role assignments for msi identity logger.debug('assigning roles to msi identity') sub_scope = '/subscriptions/{}/'.format(sub_id) cont_role = None for role in auth_client.role_definitions.list( sub_scope, filter='roleName eq \'Reader\''): cont_role = role.id break if cont_role is None: raise RuntimeError('Role Id not found for Reader') # sometimes the sp created is not added to the directory in time for # the following call, allow a minute worth of retries before giving up attempts = 0 while attempts < 30: try: role_assign = auth_client.role_assignments.create( scope=sub_scope, role_assignment_name=uuid.uuid4(), parameters=authmodels.RoleAssignmentCreateParameters( role_definition_id=cont_role, principal_id=vms[0].identity.principal_id), ) break except msrestazure.azure_exceptions.CloudError: time.sleep(2) attempts += 1 if attempts == 30: raise del attempts if settings.verbose(config): logger.debug('reader role assignment: {}'.format(role_assign)) cont_role = None for role in auth_client.role_definitions.list( sub_scope, filter='roleName eq \'Reader and Data Access\''): cont_role = role.id break if cont_role is None: raise RuntimeError('Role Id not found for Reader and Data Access') role_assign = auth_client.role_assignments.create( scope=sub_scope, role_assignment_name=uuid.uuid4(), parameters=authmodels.RoleAssignmentCreateParameters( role_definition_id=cont_role, principal_id=vms[0].identity.principal_id), ) if settings.verbose(config): logger.debug( 'reader and data access role assignment: {}'.format(role_assign)) # get ip info for vm if util.is_none_or_empty(pips): fqdn = None ipinfo = 'private_ip_address={}'.format( nics[offset].ip_configurations[0].private_ip_address) else: # refresh public ip for vm pip = network_client.public_ip_addresses.get( resource_group_name=ms.resource_group, public_ip_address_name=pips[offset].name, ) fqdn = pip.dns_settings.fqdn ipinfo = 'fqdn={} public_ip_address={}'.format(fqdn, pip.ip_address) # temporary enable port 80 for ACME challenge if fqdn is present if servconf.lets_encrypt_enabled: isr = settings.InboundNetworkSecurityRule( destination_port_range='80', source_address_prefix='*', protocol='tcp', ) logger.debug('creating temporary port 80 rule for ACME challenge') async_ops['port80'] = resource.AsyncOperation( functools.partial(resource.add_inbound_network_security_rule, network_client, ms, 'acme80', isr)) # install msi vm extension async_ops['vmext'] = {} async_ops['vmext'][0] = resource.AsyncOperation( functools.partial(resource.create_msi_virtual_machine_extension, compute_client, ms, vms[0].name, 0, settings.verbose(config)), max_retries=0, ) logger.debug('waiting for virtual machine msi extensions to provision') for offset in async_ops['vmext']: async_ops['vmext'][offset].result() # ensure port 80 rule is ready if servconf.lets_encrypt_enabled and ms.public_ip.enabled: async_ops['port80'].result() # install vm extension async_ops['vmext'][0] = resource.AsyncOperation( functools.partial(_create_virtual_machine_extension, compute_client, config, ms, bootstrap_file, blob_urls, vms[0].name, private_ips, fqdn, 0, settings.verbose(config)), max_retries=0, ) logger.debug('waiting for virtual machine extensions to provision') for offset in async_ops['vmext']: # get vm extension result vm_ext = async_ops['vmext'][offset].result() vm = vms[offset] logger.info( ('virtual machine: {} [provisioning_state={}/{} ' 'vm_size={} {}]').format(vm.id, vm.provisioning_state, vm_ext.provisioning_state, vm.hardware_profile.vm_size, ipinfo)) # disable port 80 for ACME challenge if servconf.lets_encrypt_enabled and ms.public_ip.enabled: logger.debug('removing temporary port 80 rule for ACME challenge') async_ops['port80'] = resource.AsyncOperation( functools.partial(resource.remove_inbound_network_security_rule, network_client, ms, 'acme80')) async_ops['port80'].result() # output connection info if ms.public_ip.enabled: logger.info(('To connect to Grafana, open a web browser and go ' 'to https://{}').format(fqdn)) if servconf.prometheus.port is not None: logger.info(('To connect to Prometheus, open a web browser and go ' 'to https://{}:{}').format(fqdn, servconf.prometheus.port)) else: logger.info(('To connect to Grafana, open a web browser and go ' 'to http://{} within the virtual network').format( nics[offset].ip_configurations[0].private_ip_address)) if servconf.prometheus.port is not None: logger.info( ('To connect to Prometheus, open a web browser and go ' 'to http://{}:{} within the virtual network').format( nics[offset].ip_configurations[0].private_ip_address, servconf.prometheus.port))
def create_federation_proxy(auth_client, resource_client, compute_client, network_client, blob_client, table_client, queue_client, config, resources_path, bootstrap_file, federation_files): # type: (azure.mgmt.authorization.AuthorizationManagementClient, # azure.mgmt.resource.resources.ResourceManagementClient, # azure.mgmt.compute.ComputeManagementClient, # azure.mgmt.network.NetworkManagementClient, # azure.storage.blob.BlockBlobService, # azure.cosmosdb.table.TableService, # azure.storage.queue.QueueService, # dict, pathlib.Path, Tuple[str, pathlib.Path], # List[Tuple[str, pathlib.Path]]) -> None """Create a federation proxy :param azure.mgmt.authorization.AuthorizationManagementClient auth_client: auth client :param azure.mgmt.resource.resources.ResourceManagementClient resource_client: resource client :param azure.mgmt.compute.ComputeManagementClient compute_client: compute client :param azure.mgmt.network.NetworkManagementClient network_client: network client :param azure.storage.blob.BlockBlobService blob_client: blob client :param azure.cosmosdb.table.TableService table_client: table client :param azure.storage.queue.QueueService queue_client: queue client :param dict config: configuration dict :param pathlib.Path: resources path :param Tuple[str, pathlib.Path] bootstrap_file: customscript bootstrap file :param List[Tuple[str, pathlib.Path]] federation_files: federation files """ fs = settings.federation_settings(config) # get subscription id for msi sub_id = settings.credentials_management(config).subscription_id if util.is_none_or_empty(sub_id): raise ValueError('Management subscription id not specified') # check if cluster already exists logger.debug('checking if federation proxy exists') try: vm = compute_client.virtual_machines.get( resource_group_name=fs.resource_group, vm_name=settings.generate_virtual_machine_name(fs, 0)) raise RuntimeError( 'Existing virtual machine {} found for federation proxy'.format( vm.id)) except msrestazure.azure_exceptions.CloudError as e: if e.status_code == 404: pass else: raise # confirm before proceeding if not util.confirm_action(config, 'create federation proxy'): return # create resource group if it doesn't exist resource.create_resource_group(resource_client, fs.resource_group, fs.location) # create storage containers storage.create_storage_containers_nonbatch(blob_client, table_client, queue_client, 'federation') # create file share for log persistence bs = settings.batch_shipyard_settings(config) storage.create_file_share_saskey( settings.credentials_storage( config, bs.storage_account_settings, ), '{}fedlogs'.format(bs.storage_entity_prefix), 'ingress', create_share=True, ) # create global lock storage.create_global_lock_blob(blob_client, 'federation') # upload scripts to blob storage for customscript vm extension blob_urls = storage.upload_for_nonbatch(blob_client, federation_files, 'federation') # async operation dictionary async_ops = {} # create nsg async_ops['nsg'] = resource.AsyncOperation( functools.partial(resource.create_network_security_group, network_client, fs)) # use dynamic ips for private private_ips = None logger.debug('using dynamic private ip address allocation') # create virtual network and subnet if specified vnet, subnet = resource.create_virtual_network_and_subnet( resource_client, network_client, fs.virtual_network.resource_group, fs.location, fs.virtual_network) # create public ips pips = None if fs.public_ip.enabled: async_ops['pips'] = {} async_ops['pips'][0] = resource.AsyncOperation( functools.partial(resource.create_public_ip, network_client, fs, 0)) logger.debug('waiting for public ips to provision') pips = {} for offset in async_ops['pips']: pip = async_ops['pips'][offset].result() logger.info(('public ip: {} [provisioning_state={} ip_address={} ' 'public_ip_allocation={}]').format( pip.id, pip.provisioning_state, pip.ip_address, pip.public_ip_allocation_method)) pips[offset] = pip else: logger.info('public ip is disabled for monitoring resource') # get nsg logger.debug('waiting for network security group to provision') nsg = async_ops['nsg'].result() # create nics async_ops['nics'] = {} async_ops['nics'][0] = resource.AsyncOperation( functools.partial(resource.create_network_interface, network_client, fs, subnet, nsg, private_ips, pips, 0)) # wait for nics to be created logger.debug('waiting for network interfaces to provision') nics = {} for offset in async_ops['nics']: nic = async_ops['nics'][offset].result() logger.info( ('network interface: {} [provisioning_state={} private_ip={} ' 'private_ip_allocation_method={} network_security_group={} ' 'accelerated_networking={}]').format( nic.id, nic.provisioning_state, nic.ip_configurations[0].private_ip_address, nic.ip_configurations[0].private_ip_allocation_method, nsg.name if nsg is not None else None, nic.enable_accelerated_networking)) nics[offset] = nic # read or generate ssh keys if util.is_not_empty(fs.ssh.ssh_public_key_data): key_data = fs.ssh.ssh_public_key_data else: # create universal ssh key for all vms if not specified ssh_pub_key = fs.ssh.ssh_public_key if ssh_pub_key is None: _, ssh_pub_key = crypto.generate_ssh_keypair( fs.ssh.generated_file_export_path, crypto.get_federation_ssh_key_prefix()) # read public key data with ssh_pub_key.open('rb') as fd: key_data = fd.read().decode('utf8') ssh_pub_key = compute_client.virtual_machines.models.SshPublicKey( path='/home/{}/.ssh/authorized_keys'.format(fs.ssh.username), key_data=key_data, ) # create vms async_ops['vms'] = {} async_ops['vms'][0] = resource.AsyncOperation( functools.partial(resource.create_virtual_machine, compute_client, fs, None, nics, None, ssh_pub_key, 0, enable_msi=True)) # wait for vms to be created logger.info('waiting for {} virtual machines to provision'.format( len(async_ops['vms']))) vms = {} for offset in async_ops['vms']: vms[offset] = async_ops['vms'][offset].result() logger.debug('{} virtual machines created'.format(len(vms))) # create role assignments for msi identity logger.debug('assigning roles to msi identity') sub_scope = '/subscriptions/{}/'.format(sub_id) cont_role = None for role in auth_client.role_definitions.list( sub_scope, filter='roleName eq \'Contributor\''): cont_role = role.id break if cont_role is None: raise RuntimeError('Role Id not found for Reader') # sometimes the sp created is not added to the directory in time for # the following call, allow some retries before giving up attempts = 0 while attempts < 90: try: role_assign = auth_client.role_assignments.create( scope=sub_scope, role_assignment_name=uuid.uuid4(), parameters=authmodels.RoleAssignmentCreateParameters( role_definition_id=cont_role, principal_id=vms[0].identity.principal_id), ) break except msrestazure.azure_exceptions.CloudError: time.sleep(2) attempts += 1 if attempts == 90: raise del attempts if settings.verbose(config): logger.debug('reader role assignment: {}'.format(role_assign)) cont_role = None for role in auth_client.role_definitions.list( sub_scope, filter='roleName eq \'Reader and Data Access\''): cont_role = role.id break if cont_role is None: raise RuntimeError('Role Id not found for Reader and Data Access') role_assign = auth_client.role_assignments.create( scope=sub_scope, role_assignment_name=uuid.uuid4(), parameters=authmodels.RoleAssignmentCreateParameters( role_definition_id=cont_role, principal_id=vms[0].identity.principal_id), ) if settings.verbose(config): logger.debug( 'reader and data access role assignment: {}'.format(role_assign)) # get ip info for vm if util.is_none_or_empty(pips): fqdn = None ipinfo = 'private_ip_address={}'.format( nics[offset].ip_configurations[0].private_ip_address) else: # refresh public ip for vm pip = network_client.public_ip_addresses.get( resource_group_name=fs.resource_group, public_ip_address_name=pips[offset].name, ) fqdn = pip.dns_settings.fqdn ipinfo = 'fqdn={} public_ip_address={}'.format(fqdn, pip.ip_address) for offset in async_ops['vmext']: async_ops['vmext'][offset].result() # install vm extension async_ops['vmext'] = {} async_ops['vmext'][0] = resource.AsyncOperation( functools.partial(_create_virtual_machine_extension, compute_client, config, fs, bootstrap_file, blob_urls, vms[0].name, private_ips, fqdn, 0, settings.verbose(config)), max_retries=0, ) logger.debug('waiting for virtual machine extensions to provision') for offset in async_ops['vmext']: # get vm extension result vm_ext = async_ops['vmext'][offset].result() vm = vms[offset] logger.info( ('virtual machine: {} [provisioning_state={}/{} ' 'vm_size={} {}]').format(vm.id, vm.provisioning_state, vm_ext.provisioning_state, vm.hardware_profile.vm_size, ipinfo))