def create_or_update_resource_group(self, resource_group_name: str, location: str): """Function that creates or updates a resource group :param resource_group_name: Resource group name to analyze :type resource_group_name: str :param location: Resource group location :type location: str :raises: :class:`Exception` """ if location is None or location == '': raise CustomException('No location has been set') try: resource_group_parameters = \ ResourceGroup(location = location) new_resource_group: ResourceGroup = \ self._resource_management_client\ .resource_groups\ .create_or_update( resource_group_name, resource_group_parameters) self._logger.info( 'resource group {} created on {} location'.format( new_resource_group.name, new_resource_group.location)) except Exception as e: self._logger.error( 'The following error occurred while creating a resource group: ' + str(e)) self._logger.error(e) sys.exit( 1 ) # Exit the module as it is not possible to progress the deployment. raise (e)
def deploy_template_to_new_resource_group(self, template_url, group_name, parameters): deployment_name = DEPLOYMENT_NAME.format(group_name) # Resource group must be created before validation can occur if self.rmc.resource_groups.check_existence(group_name): raise Exception( "Group name already exists / taken: {}".format(group_name)) log.info('Starting resource group_creation') def get_all_details(error): formatted_message = '{}: {}\n\n'.format(error.code, error.message) if error.details is None: return formatted_message for d in error.details: formatted_message += get_all_details(d) return formatted_message with contextlib.ExitStack() as stack: self.rmc.resource_groups.create_or_update( group_name, ResourceGroup(location=self.location)) # Ensure the resource group will be deleted if the following steps fail stack.callback(self.rmc.resource_groups.delete, group_name) log.info('Resource group created: {}'.format(group_name)) deployment_properties = self.create_deployment_properties( template_url, parameters) log.info('Checking with Azure to validate template deployment') result = self.rmc.deployments.validate( group_name, deployment_name, properties=deployment_properties) if result.error: raise Exception("Template verification failed!\n{}".format( get_all_details(result.error))) log.info('Template successfully validated') log.info('Starting template deployment') self.rmc.deployments.create_or_update(group_name, deployment_name, deployment_properties) stack.pop_all() log.info('Successfully started template deployment')
def main(): validate_env() location = os.getenv('AZURE_LOCATION', 'East US') credentials = azure.common.credentials.ServicePrincipalCredentials( client_id=os.environ['AZURE_CLIENT_ID'], secret=os.environ['AZURE_CLIENT_SECRET'], tenant=os.environ['AZURE_TENANT_ID']) subscription_id = os.environ['AZURE_SUBSCRIPTION_ID'] template = TemplateLink(uri=os.environ['AZURE_TEMPLATE_URL']) # tenant_id = os.environ.get('AZURE_TENANT_ID') # client_id = os.environ.get('AZURE_CLIENT_ID') # client_secret = os.environ.get('AZURE_CLIENT_SECRET') group_name = 'testing' + ''.join(random.choice('01234567890abcdef') for n in range(10)) deployment_name = 'deployment{}'.format(uuid.uuid4().hex) rmc = ResourceManagementClient(credentials, subscription_id) template_parameters = get_env_params() # Output resource group print("Resource group name: {}".format(group_name)) print("Deployment name: {}".format(deployment_name)) azure_cluster = { 'resource_group_name': group_name, 'deployment_name': deployment_name} pkgpanda.util.write_json('azure-cluster.json', azure_cluster) # Create a new resource group print("Creating new resource group in location: {}".format(location)) if rmc.resource_groups.check_existence(group_name): print("ERROR: Group name already exists / taken: {}".format(group_name)) rmc.resource_groups.create_or_update( group_name, ResourceGroup(location=location)) test_successful = False try: deployment_properties = DeploymentProperties( template_link=template, mode=DeploymentMode.incremental, parameters=template_parameters) # Use RPC against azure to validate the ARM template is well-formed result = rmc.deployments.validate(group_name, deployment_name, properties=deployment_properties) if result.error: print("Template verification failed\n{}".format(result.error), file=sys.stderr) sys.exit(1) # Actually create a template deployment print("Creating template deployment ...") deploy_poller = rmc.deployments.create_or_update(group_name, deployment_name, deployment_properties) # Stop after 45 attempts (each one takes up to one minute) @retry(stop_max_attempt_number=45) def poll_deploy(): res = deploy_poller.result(timeout=60) print("Current deploy state: {}".format(res.properties.provisioning_state)) assert deploy_poller.done(), "Not done deploying." print("Waiting for template to deploy ...") try: poll_deploy() except: print("Current deploy status:\n{}".format(deploy_poller.result(0))) raise print("Template deployed successfully") assert deploy_poller.done(), "Deployment failed / polling didn't reach deployment done." deployment_result = deploy_poller.result() print(deployment_result.properties.outputs) master_lb = deployment_result.properties.outputs['masterFQDN']['value'] print("Template deployed using SSH private key: https://mesosphere.onelogin.com/notes/18444") print("For troubleshooting, master0 can be reached using: ssh -p 2200 {}@{}".format( get_value('linuxAdminUsername'), master_lb)) # Run test now, so grab IPs nmc = NetworkManagementClient(credentials, subscription_id) ip_buckets = { 'master': [], 'private': [], 'public': []} for resource in rmc.resource_groups.list_resources( group_name, filter=("resourceType eq 'Microsoft.Network/networkInterfaces' or " "resourceType eq 'Microsoft.Compute/virtualMachineScaleSets'")): if resource.type == 'Microsoft.Network/networkInterfaces': nics = [nmc.network_interfaces.get(group_name, resource.name)] elif resource.type == 'Microsoft.Compute/virtualMachineScaleSets': nics = list(nmc.network_interfaces.list_virtual_machine_scale_set_network_interfaces( virtual_machine_scale_set_name=resource.name, resource_group_name=group_name)) else: raise('Unexpected resourceType: {}'.format(resource.type)) for bucket_name in ip_buckets.keys(): if bucket_name in resource.name: for n in nics: for config in n.ip_configurations: ip_buckets[bucket_name].append(config.private_ip_address) print('Detected IP configuration: {}'.format(ip_buckets)) with Tunnel(get_value('linuxAdminUsername'), 'ssh_key', master_lb, port=2200) as t: integration_test( tunnel=t, test_dir='/home/{}'.format(get_value('linuxAdminUsername')), dcos_dns=ip_buckets['master'][0], master_list=ip_buckets['master'], agent_list=ip_buckets['private'], public_agent_list=ip_buckets['public'], provider='azure', test_dns_search=False, add_env=get_test_config(), pytest_cmd=os.getenv('DCOS_PYTEST_CMD', "py.test -vv -s -rs -m 'not ccm' ") + os.getenv('CI_FLAGS', '')) test_successful = True except Exception as ex: traceback.print_exc() print("ERROR: exception {}".format(ex)) raise finally: if os.getenv('AZURE_CLEANUP') == 'false': print("Cluster must be cleaned up manually") print("Cluster details: {}".format(azure_cluster)) else: # Send a delete request # TODO(cmaloney): The old code had a retry around this: # @retry(wait_exponential_multiplier=1000, wait_exponential_max=60*1000, stop_max_delay=(30*60*1000)) poller = rmc.resource_groups.delete(group_name) # poll for the delete to complete print("Deleting resource group: {} ...".format(group_name)) @retry(wait_fixed=(5 * 1000), stop_max_delay=(60 * 60 * 1000)) def wait_for_delete(): assert poller.done(), "Timed out waiting for delete" print("Waiting for delete ...") wait_for_delete() print("Clean up successful") if test_successful: print("Azure test deployment succeeded") else: print("ERROR: Azure test deployment failed", file=sys.stderr) sys.exit(2)
def deploy_template(self): """ Deploy the targeted template and parameters :param module: Ansible module containing the validated configuration for the deployment template :param client: resource management client for azure :param conn_info: connection info needed :return: """ deploy_parameter = DeploymentProperties(self.deployment_mode) if not self.parameters_link: deploy_parameter.parameters = self.parameters else: deploy_parameter.parameters_link = ParametersLink( uri=self.parameters_link) if not self.template_link: deploy_parameter.template = self.template else: deploy_parameter.template_link = TemplateLink( uri=self.template_link) params = ResourceGroup(location=self.location, tags=self.tags) try: self.rm_client.resource_groups.create_or_update( self.resource_group_name, params) except CloudError as exc: self.fail( "Resource group create_or_update failed with status code: %s and message: %s" % (exc.status_code, exc.message)) try: result = self.rm_client.deployments.create_or_update( self.resource_group_name, self.deployment_name, deploy_parameter) deployment_result = self.get_poller_result(result) if self.wait_for_deployment_completion: while deployment_result.properties is None or deployment_result.properties.provisioning_state not in [ 'Canceled', 'Failed', 'Deleted', 'Succeeded' ]: time.sleep(self.wait_for_deployment_polling_period) deployment_result = self.rm_client.deployments.get( self.resource_group_name, self.deployment_name) except CloudError as exc: failed_deployment_operations = self._get_failed_deployment_operations( self.deployment_name) self.log("Deployment failed %s: %s" % (exc.status_code, exc.message)) self.fail( "Deployment failed with status code: %s and message: %s" % (exc.status_code, exc.message), failed_deployment_operations=failed_deployment_operations) if self.wait_for_deployment_completion and deployment_result.properties.provisioning_state != 'Succeeded': self.log("provisioning state: %s" % deployment_result.properties.provisioning_state) failed_deployment_operations = self._get_failed_deployment_operations( self.deployment_name) self.fail( 'Deployment failed. Deployment id: %s' % deployment_result.id, failed_deployment_operations=failed_deployment_operations) return deployment_result
def run(): location = os.getenv('AZURE_LOCATION', 'East US') credentials = azure.common.credentials.ServicePrincipalCredentials( client_id=os.environ['AZURE_CLIENT_ID'], secret=os.environ['AZURE_CLIENT_SECRET'], tenant=os.environ['AZURE_TENANT_ID']) subscription_id = os.environ['AZURE_SUBSCRIPTION_ID'] template = TemplateLink(uri=os.environ['AZURE_TEMPLATE_URL']) # tenant_id = os.environ.get('AZURE_TENANT_ID') # client_id = os.environ.get('AZURE_CLIENT_ID') # client_secret = os.environ.get('AZURE_CLIENT_SECRET') group_name = 'tesing' + ''.join( random.choice('01234567890abcdef') for n in range(10)) deployment_name = 'deployment{}'.format(uuid.uuid4().hex) rmc = ResourceManagementClient(credentials, subscription_id) template_parameters = get_env_params() # Output resource group print("Resource group name: {}".format(group_name)) print("Deployment name: {}".format(deployment_name)) azure_cluster = { 'resource_group_name': group_name, 'deployment_name': deployment_name } pkgpanda.util.write_json('azure-cluster.json', azure_cluster) # Create a new resource group print("Creating new resource group in location: {}".format(location)) if rmc.resource_groups.check_existence(group_name): print( "ERROR: Group name already exists / taken: {}".format(group_name)) rmc.resource_groups.create_or_update(group_name, ResourceGroup(location=location)) test_successful = False try: deployment_properties = DeploymentProperties( template_link=template, mode=DeploymentMode.incremental, parameters=template_parameters) # Use RPC against azure to validate the ARM template is well-formed result = rmc.deployments.validate(group_name, deployment_name, properties=deployment_properties) if result.error: print("Template verification failed\n{}".format(result.error), file=sys.stderr) sys.exit(1) # Actually create a template deployment print("Creating template deployment ...") deploy_poller = rmc.deployments.create_or_update( group_name, deployment_name, deployment_properties) # Stop after 45 attempts (each one takes up to one minute) @retry(stop_max_attempt_number=45) def poll_deploy(): res = deploy_poller.result(timeout=60) print("Current deploy state: {}".format( res.properties.provisioning_state)) assert deploy_poller.done(), "Not done deploying." print("Waiting for template to deploy ...") try: poll_deploy() except: print("Current deploy status:\n{}".format(deploy_poller.result(0))) raise print("Template deployed successfully") assert deploy_poller.done( ), "Deployment failed / polling didn't reach deployment done." deployment_result = deploy_poller.result() print(deployment_result.properties.outputs) master_lb = deployment_result.properties.outputs['dnsAddress']['value'] master_url = "http://{}".format(master_lb) print( "Template deployed using SSH private key: https://mesosphere.onelogin.com/notes/18444" ) print( "For troubleshooting, master0 can be reached using: ssh -p 2200 core@{}" .format(master_lb)) @retry(wait_fixed=(5 * 1000), stop_max_delay=(15 * 60 * 1000)) def poll_on_dcos_ui_up(): r = get_dcos_ui(master_url) assert r is not None and r.status_code == requests.codes.ok, \ "Unable to reach DC/OS UI: {}".format(master_url) print("Waiting for DC/OS UI at: {} ...".format(master_url)) poll_on_dcos_ui_up() test_successful = True except Exception as ex: print("ERROR: exception {}".format(ex)) raise finally: # Send a delete request # TODO(cmaloney): The old code had a retry around this: # @retry(wait_exponential_multiplier=1000, wait_exponential_max=60*1000, stop_max_delay=(30*60*1000)) poller = rmc.resource_groups.delete(group_name) # poll for the delete to complete print("Deleting resource group: {} ...".format(group_name)) @retry(wait_fixed=(5 * 1000), stop_max_delay=(60 * 60 * 1000)) def wait_for_delete(): assert poller.done(), "Timed out waiting for delete" print("Waiting for delete ...") wait_for_delete() print("Clean up successful") if test_successful: print("Azure test deployment succeeded") else: print("ERROR: Azure test deployment failed", file=sys.stderr) sys.exit(2)
def main(): module = AnsibleModule( argument_spec = dict( client_id=dict(required=False), client_secret=dict(required=False), tenant_id=dict(required=False), subscription_id=dict(required=False), profile=dict(required=False), ad_user=dict(required=False), password=dict(required=False), src_json = dict(), resource_group_name = dict(required=True), resource_group_location = dict(), resource_url = dict(required=True), state = dict(default='present', choices=['absent', 'present']), force = dict(required=False, default=False, type='bool'), check_existing = dict(required=False, default=True, type='bool'), ), # Implementing check-mode using HEAD is impossible, since size/date is not 100% reliable supports_check_mode = False, ) creds_params = {} logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)s %(message)s', filename='/tmp/ansible_arm.log', filemode='w') if not HAS_ARM: module.fail_json(msg='azure python sdk required for this module') if module.params['client_id']: creds_params['client_id'] = module.params.get('client_id') if module.params['client_secret']: creds_params['client_secret'] = module.params.get('client_secret') if module.params['tenant_id']: creds_params['tenant_id'] = module.params.get('tenant_id') if module.params['subscription_id']: creds_params['subscription_id'] = module.params.get('subscription_id') if module.params['profile']: profile = module.params.get('profile') else: profile = None if module.params['ad_user']: creds_params['ad_user'] = module.params.get('ad_user') if module.params['password']: creds_params['password'] = module.params.get('password') if module.params['src_json']: src_json = module.params.get('src_json') else: src_json = 'none' resource_group_name = module.params.get('resource_group_name') resource_url = module.params.get('resource_url') if module.params['resource_group_location']: resource_group_location = module.params.get('resource_group_location') else: resource_group_location = 'none' force = module.params['force'] check_existing = module.params['check_existing'] url_method = 'put' #try: #authenticate to azure creds = None if profile: path = expanduser("~/.azure/credentials") try: config = ConfigParser.SafeConfigParser() config.read(path) except Exception as exc: module.fail_json("Failed to access {0}. Check that the file exists and you have read access. {1}".format(path, str(exc))) if not config.has_section(profile): module.fail_json("Config file does not appear to have section " + profile) for key, val in config.items(profile): creds_params[key] = val if 'client_id' in creds_params and 'client_secret' in creds_params: endpoint = 'https://login.microsoftonline.com/' + creds_params['tenant_id'] + '/oauth2/token' auth_token = get_token_from_client_credentials( endpoint=endpoint, client_id=creds_params['client_id'], client_secret=creds_params['client_secret'], ) creds = ServicePrincipalCredentials(client_id=creds_params['client_id'], secret=creds_params['client_secret'], tenant=creds_params['tenant_id']) elif 'ad_user' in creds_params and 'password' in creds_params: creds = UserPassCredentials(creds_params['ad_user'], creds_params['password']) auth_token = creds.token['access_token'] # at this point, we should have creds and a subscription id if not creds: module.fail_json(msg="Unable to login to Azure with the current parameters/options") if not creds_params['subscription_id']: module.fail_json(msg="Unable to select a working Azure subscription given the current parameters/options") #construct resource client resource_client = ResourceManagementClient(credentials=creds, subscription_id=creds_params['subscription_id']) #Check rg try: rg_list_result = resource_client.resource_groups.get(resource_group_name) rg_does_exist = 'True' except: rg_does_exist = 'False' #Create RG if necessary if module.params['state'] == 'present': if (rg_does_exist == 'False'): if (resource_group_location == 'none'): module.fail_json(msg='Resource group does not exist, and resource_group_location isnt specified') result = resource_client.resource_groups.create_or_update( resource_group_name, ResourceGroup( location=resource_group_location, ), ) #read template file and params file if src_json != 'none': jsonfilefile = open(src_json) jsonpayload = jsonfilefile.read() jsonfilefile.close() else: jsonpayload = None url = "https://management.azure.com/subscriptions/" + creds_params['subscription_id'] + "/resourceGroups/" + resource_group_name + "/" + resource_url logging.info(str.format("Testing if resource already exists: {0}", url)) headers = { 'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': 'Bearer ' + auth_token } class Object(object): pass returnobj = Object() #Check if the resource exists result = None exception_message = "" if check_existing is False: does_exist = False else: does_exist_request = requests.get(url, headers=headers) if does_exist_request.status_code in (400, 404): does_exist = False else: does_exist = True if ((does_exist is False) and (module.params['state'] is 'present')) or ((does_exist is True) and (force is True) and (module.params['state'] is 'present')): try: if src_json is 'none': result = requests.put(url, headers=headers) else: result = requests.put(url, headers=headers, data=jsonpayload) except Exception as e: exception_message = str(e) if (does_exist == False) and (module.params['state'] == 'absent'): module.exit_json(changed=False, status_code=None, url=url) if (does_exist == True) and (module.params['state'] == 'present'): module.exit_json(changed=False, status_code=does_exist_request.status_code, url=url, content=does_exist_request.json()) if (does_exist == True) and (module.params['state'] == 'absent'): result = requests.delete(url, headers=headers) if not result: module.fail_json(msg="We shouldn't be here. Exception: " + exception_message) returnobj.url = url if result.status_code in (200, 201, 202): returnobj.changed = True if result.headers.get('azure-asyncoperation'): # This is an async operation, pull until done async_url = result.headers.get('azure-asyncoperation') currentOp = 'updating' while currentOp == 'updating': print(currentOp) result = requests.get(async_url, headers=headers) currentStatus = json.loads(result.text)['status'] if str(currentStatus).lower() == 'failed': currentOp = "failed" errorMsg = json.loads(result.text)['error']['message'] module.fail_json(msg=errorMsg, status_code=result.status_code, url=url) elif str(currentStatus).lower() == 'inprogress': currentOp = 'updating' time.sleep(0.5) elif str(currentStatus).lower() == 'succeeded': currentOp = 'succeeded' module.exit_json(changed=True, status_code=result.status_code, url=url, content=result.json()) else: pass elif result.status_code == 202 and result.headers.get('location'): async_url = result.headers.get('location') currentOp = 'updating' while currentOp == 'updating': result = requests.get(async_url, headers=headers) if result.status_code == 202: # Operation not finished time.sleep(0.5) elif result.status_code == 200: currentOp = 'succeeded' else: module.fail_json(msg='Error while waiting for resource to be provisioned', status_code=result.status_code, url=async_url) module.exit_json(changed=True, status_code=result.status_code, url=url, content=result.json()) elif result.status_code == 204: module.exit_json(changed=True, status_code=result.status_code, url=url) else: module.fail_json(msg='Error', status_code=result.status_code, url=url) module.exit_json(changed=True, status=result.text, url=url)
def _create_resource_group(cmd, name, location): client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES) params = ResourceGroup(location=location) client.resource_groups.create_or_update(name, params)
def _process_db_up_namespace(cmd, namespace, db_type=None): # populate from cache if existing when no resource group name provided resource_client = resource_client_factory(cmd.cli_ctx) if namespace.resource_group_name is None: _set_value(db_type, namespace, 'resource_group_name', 'group', cache=False) try: resource_client.resource_groups.get(namespace.resource_group_name) except: # Clear resource group name information when it is invalid namespace.resource_group_name = None # populate from cache if existing when no location provided if namespace.location is None: _set_value(db_type, namespace, 'location', 'location', cache=False) # generate smart defaults when namespace.location is None if _get_value(db_type, namespace, 'location', 'location') is None: try: get_default_location_from_resource_group(cmd, namespace) except (CLIError, ValidationError): namespace.location = 'eastus' _set_value(db_type, namespace, 'location', 'location', default=namespace.location) # When resource group name in namespace is different from what in cache, reset it and create new server name if namespace.resource_group_name != get_config_value( db_type, 'group', None): set_config_value(db_type, 'group', namespace.resource_group_name) if namespace.server_name is None: namespace.server_name = create_random_resource_name('server') set_config_value(db_type, 'server', namespace.server_name) # When no resource group name in namespace and cache, create new resource group with random name create_resource_group = True if namespace.resource_group_name is None: # No resource group provided and in cache namespace.resource_group_name = create_random_resource_name('group') else: try: resource_client.resource_groups.get(namespace.resource_group_name) create_resource_group = False except CloudError: # throw exception when resource group name is invalid pass if create_resource_group: # create new resource group params = ResourceGroup(location=namespace.location) logger.warning('Creating Resource Group \'%s\'...', namespace.resource_group_name) resource_client.resource_groups.create_or_update( namespace.resource_group_name, params) _set_value(db_type, namespace, 'resource_group_name', 'group', default=namespace.resource_group_name) _set_value(db_type, namespace, 'server_name', 'server', default=create_random_resource_name('server')) _set_value(db_type, namespace, 'administrator_login', 'login', default=generate_username()) if namespace.generate_password: namespace.administrator_login_password = str(uuid.uuid4()) del namespace.generate_password _set_value(db_type, namespace, 'database_name', 'database', default=DEFAULT_DATABASE_NAME)
def __init__(self, azure_config, skip_setup=False): self.config = azure_config subscription_id = azure_config.subscription_id username = azure_config.username password = azure_config.password group_name = azure_config.group_name storage_name = azure_config.storage_name virtual_network_name = azure_config.virtual_network_name subnet_name = azure_config.subnet_name region = azure_config.region self.vms = {} # 0. Authentication credentials = UserPassCredentials(username, password) self.resource_client = ResourceManagementClient( credentials, subscription_id) self.storage_client = StorageManagementClient(credentials, subscription_id) self.network_client = NetworkManagementClient(credentials, subscription_id) self.compute_client = ComputeManagementClient(credentials, subscription_id) if not skip_setup: # 1. Create a resource group result = self.resource_client.resource_groups.create_or_update( group_name, ResourceGroup(location=region), ) # 2. Create a storage account result = self.storage_client.storage_accounts.create( group_name, storage_name, azure.mgmt.storage.models.StorageAccountCreateParameters( location=region, account_type=azure.mgmt.storage.models.AccountType. standard_lrs, ), ) result.wait() # 3. Create a virtual network result = self.network_client.virtual_networks.create_or_update( group_name, virtual_network_name, azure.mgmt.network.models.VirtualNetwork( location=region, address_space=azure.mgmt.network.models.AddressSpace( address_prefixes=[ '10.0.0.0/16', ], ), subnets=[ azure.mgmt.network.models.Subnet( name=subnet_name, address_prefix='10.0.0.0/24', ), ], ), ) result.wait()
def run(): location = os.getenv('AZURE_LOCATION', 'East US') credentials = azure.common.credentials.ServicePrincipalCredentials( client_id=os.environ['AZURE_CLIENT_ID'], secret=os.environ['AZURE_CLIENT_SECRET'], tenant=os.environ['AZURE_TENANT_ID']) subscription_id = os.environ['AZURE_SUBSCRIPTION_ID'] template = TemplateLink(uri=os.environ['AZURE_TEMPLATE_URL']) # tenant_id = os.environ.get('AZURE_TENANT_ID') # client_id = os.environ.get('AZURE_CLIENT_ID') # client_secret = os.environ.get('AZURE_CLIENT_SECRET') group_name = 'tesing' + ''.join( random.choice('01234567890abcdef') for n in range(10)) deployment_name = 'deployment{}'.format(uuid.uuid4().hex) rmc = ResourceManagementClient(credentials, subscription_id) template_parameters = get_env_params() if template_parameters.get('numberOfPrivateSlaves'): assert template_parameters['numberOfPrivateSlaves'][ 'value'] >= 2, 'Test requires at least 2 private slaves!' else: template_parameters['numberOfPrivateSlaves'] = {'value': 2} if template_parameters.get('numberOfPublicSlaves'): assert template_parameters['numberOfPublicSlaves'][ 'value'] >= 1, 'Test requires at least 1 public slave!' else: template_parameters['numberOfPublicSlaves'] = {'value': 1} # Output resource group print("Resource group name: {}".format(group_name)) print("Deployment name: {}".format(deployment_name)) azure_cluster = { 'resource_group_name': group_name, 'deployment_name': deployment_name } pkgpanda.util.write_json('azure-cluster.json', azure_cluster) # Create a new resource group print("Creating new resource group in location: {}".format(location)) if rmc.resource_groups.check_existence(group_name): print( "ERROR: Group name already exists / taken: {}".format(group_name)) rmc.resource_groups.create_or_update(group_name, ResourceGroup(location=location)) test_successful = False try: deployment_properties = DeploymentProperties( template_link=template, mode=DeploymentMode.incremental, parameters=template_parameters) # Use RPC against azure to validate the ARM template is well-formed result = rmc.deployments.validate(group_name, deployment_name, properties=deployment_properties) if result.error: print("Template verification failed\n{}".format(result.error), file=sys.stderr) sys.exit(1) # Actually create a template deployment print("Creating template deployment ...") deploy_poller = rmc.deployments.create_or_update( group_name, deployment_name, deployment_properties) # Stop after 45 attempts (each one takes up to one minute) @retry(stop_max_attempt_number=45) def poll_deploy(): res = deploy_poller.result(timeout=60) print("Current deploy state: {}".format( res.properties.provisioning_state)) assert deploy_poller.done(), "Not done deploying." print("Waiting for template to deploy ...") try: poll_deploy() except: print("Current deploy status:\n{}".format(deploy_poller.result(0))) raise print("Template deployed successfully") assert deploy_poller.done( ), "Deployment failed / polling didn't reach deployment done." deployment_result = deploy_poller.result() print(deployment_result.properties.outputs) master_lb = deployment_result.properties.outputs['dnsAddress']['value'] master_url = "http://{}".format(master_lb) print( "Template deployed using SSH private key: https://mesosphere.onelogin.com/notes/18444" ) print( "For troubleshooting, master0 can be reached using: ssh -p 2200 core@{}" .format(master_lb)) @retry(wait_fixed=(5 * 1000), stop_max_delay=(15 * 60 * 1000)) def poll_on_dcos_ui_up(): r = get_dcos_ui(master_url) assert r is not None and r.status_code == requests.codes.ok, \ "Unable to reach DC/OS UI: {}".format(master_url) print("Waiting for DC/OS UI at: {} ...".format(master_url)) poll_on_dcos_ui_up() # Run test now, so grab IPs nmc = NetworkManagementClient(credentials, subscription_id) ip_buckets = { 'masterNodeNic': [], 'slavePrivateNic': [], 'slavePublicNic': [] } for resource in rmc.resource_groups.list_resources(group_name): for bucket_name, bucket in ip_buckets.items(): if resource.name.startswith(bucket_name): nic = nmc.network_interfaces.get(group_name, resource.name) all_ips = [] for config in nic.ip_configurations: all_ips.append(config.private_ip_address) bucket.extend(all_ips) with closing(SSHTunnel('core', 'ssh_key', master_lb, port=2200)) as t: integration_test( tunnel=t, test_dir='/home/core', dcos_dns=master_lb, master_list=ip_buckets['masterNodeNic'], agent_list=ip_buckets['slavePrivateNic'], public_agent_list=ip_buckets['slavePublicNic'], provider='azure', test_dns_search=False, pytest_dir=os.getenv( 'DCOS_PYTEST_DIR', '/opt/mesosphere/active/dcos-integration-test'), pytest_cmd=os.getenv('DCOS_PYTEST_CMD', "py.test -vv -m 'not ccm' ") + os.getenv('CI_FLAGS', '')) test_successful = True except Exception as ex: print("ERROR: exception {}".format(ex)) raise finally: # Send a delete request # TODO(cmaloney): The old code had a retry around this: # @retry(wait_exponential_multiplier=1000, wait_exponential_max=60*1000, stop_max_delay=(30*60*1000)) poller = rmc.resource_groups.delete(group_name) # poll for the delete to complete print("Deleting resource group: {} ...".format(group_name)) @retry(wait_fixed=(5 * 1000), stop_max_delay=(60 * 60 * 1000)) def wait_for_delete(): assert poller.done(), "Timed out waiting for delete" print("Waiting for delete ...") wait_for_delete() print("Clean up successful") if test_successful: print("Azure test deployment succeeded") else: print("ERROR: Azure test deployment failed", file=sys.stderr) sys.exit(2)
def deploy_template(module, client, conn_info): """ Deploy the targeted template and parameters :param module: Ansible module containing the validated configuration for the deployment template :param client: resource management client for azure :param conn_info: connection info needed :return: """ deployment_name = conn_info["deployment_name"] group_name = conn_info["resource_group_name"] deploy_parameter = DeploymentProperties() deploy_parameter.mode = module.params.get('deployment_mode') if module.params.get('parameters_link') is None: deploy_parameter.parameters = module.params.get('parameters') else: parameters_link = ParametersLink( uri=module.params.get('parameters_link')) deploy_parameter.parameters_link = parameters_link if module.params.get('template_link') is None: deploy_parameter.template = module.params.get('template') else: template_link = TemplateLink(uri=module.params.get('template_link')) deploy_parameter.template_link = template_link params = ResourceGroup(location=module.params.get('location'), tags=module.params.get('tags')) try: client.resource_groups.create_or_update(group_name, params) result = client.deployments.create_or_update(group_name, deployment_name, deploy_parameter) deployment_result = result.result( ) # Blocking wait, return the Deployment object if module.params.get('wait_for_deployment_completion'): while not deployment_result.properties.provisioning_state in [ 'Canceled', 'Failed', 'Deleted', 'Succeeded' ]: deployment_result = client.deployments.get( group_name, deployment_name) time.sleep( module.params.get('wait_for_deployment_polling_period')) if deployment_result.properties.provisioning_state == 'Succeeded': return deployment_result failed_deployment_operations = get_failed_deployment_operations( module, client, group_name, deployment_name) module.fail_json( msg='Deployment failed. Deployment id: %s' % (deployment_result.id), failed_deployment_operations=failed_deployment_operations) except CloudError as e: failed_deployment_operations = get_failed_deployment_operations( module, client, group_name, deployment_name) module.fail_json( msg='Deploy create failed with status code: %s and message: "%s"' % (e.status_code, e.message), failed_deployment_operations=failed_deployment_operations)
################################################# # Create management clients rc = ResourceManagementClient(credentials=CREDENTIALS, subscription_id=SUBSCRIPTION_ID) ################################################# # Create a resource group # # A resource group contains our entire deployment # and makes it easy to manage related services. print("Creating resource group:", RESOURCE_GROUP) rc.resource_groups.create_or_update(RESOURCE_GROUP, ResourceGroup(location=LOCATION)) try: ################################################# # Create a resource manager template # # The template defines our entire service, including # the storage account and website. After deployment # is complete, our site is ready to use. # # Available arguments for templates can be found # at http://aka.ms/arm-template and http://resources.azure.com print("Deploying:", DEPLOYMENT)
def create_or_update_workspace(auth, resource_group_name, workspace_name, subscription_id, location=None, create_resource_group=None, sku='basic', tags=None, friendly_name=None, storage_account=None, key_vault=None, app_insights=None, containerRegistry=None, adb_workspace=None, primary_user_assigned_identity=None, cmk_keyvault=None, resource_cmk_uri=None, hbi_workspace=False, default_cpu_compute_target=None, default_gpu_compute_target=None, private_endpoint_config=None, private_endpoint_auto_approval=None, exist_ok=False, show_output=True, user_assigned_identity_for_cmk_encryption=None, system_datastores_auth_mode='accesskey', is_update_dependent_resources=False): """ Create or update workspace :param auth: :type auth: azureml.core.authentication.AbstractAuthentication :param resource_group_name: :param workspace_name: :param subscription_id: :type subscription_id: str :param location: :param sku: The workspace SKU - basic or enterprise :type sku: str :param friendly_name: :type container_registry: str :param adb_workspace: the adb workspace associated with aml workspace :type adb_workspace: str :param primary_user_assigned_identity: The resource id of the user assigned identity that used to represent the workspace :type primary_user_assigned_identity: str :param cmk_keyvault: The key vault containing the customer managed key in the Azure resource ID format. :type cmk_keyvault: str :param resource_cmk_uri: The key URI of the customer managed key to encrypt the data at rest. :type resource_cmk_uri: str :param hbi_workspace: Specifies whether the customer data is of High Business Impact(HBI), i.e., contains sensitive business information. :type hbi_workspace: bool :param default_cpu_compute_target: A configuration that will be used to create a CPU compute. If None, no compute will be created. :type default_cpu_compute_target: azureml.core.AmlCompute.AmlComputeProvisioningConfiguration :param default_gpu_compute_target: A configuration that will be used to create a GPU compute. If None, no compute will be created. :param user_assigned_identity_for_cmk_encryption: The resource id of the user assigned identity that needs to be used to access the customer manage key :type user_assigned_identity_for_cmk_encryption: str :return: :rtype: azureml._base_sdk_common.workspace.models.workspace.Workspace :param system_datastores_auth_mode: Determines whether or not to use credentials for the system datastores of the workspace 'workspaceblobstore' and 'workspacefilestore'. The default value is 'accessKey', in which case, the workspace will create the system datastores with credentials. If set to 'identity', the workspace will create the system datastores with no credentials. :type system_datastores_auth_mode: str """ vnet = _validate_private_endpoint(auth, private_endpoint_config, location) vnet_location = None if vnet: vnet_location = vnet.location resource_management_client = resource_client_factory(auth, subscription_id) found = True try: resource_management_client.resource_groups.get(resource_group_name) except CloudError as e: if e.status_code == 404: # Resource group not found case. found = False else: from azureml._base_sdk_common.common import get_http_exception_response_string raise WorkspaceException(get_http_exception_response_string(e.response)) if not found: if not location: raise WorkspaceException("Resource group was not found and location was not provided. " "Provide location with --location to create a new resource group.") else: rg_location = location if create_resource_group is None: # Flag was not set, we need to prompt user for creation of resource group give_warning("UserWarning: The resource group doesn't exist or was not provided. " "AzureML SDK can create a resource group={} in location={} " "using subscription={}. Press 'y' to confirm".format(resource_group_name, rg_location, subscription_id)) yes_set = ['yes', 'y', 'ye', ''] choice = input().lower() if choice in yes_set: create_resource_group = True else: raise WorkspaceException("Resource group was not found and " "confirmation prompt was denied.") if create_resource_group: # Create the required resource group, give the user details about it. give_warning("UserWarning: The resource group doesn't exist or was not provided. " "AzureML SDK is creating a resource group={} in location={} " "using subscription={}.".format(resource_group_name, rg_location, subscription_id)) from azure.mgmt.resource.resources.models import ResourceGroup rg_tags = {"creationTime": str(time.time()), "creationSource": "azureml-sdk"} # Adding location as keyworded argument for compatibility with azure-mgmt-resource 1.2.* # and azure-mgmt-resource 2.0.0 resource_group_properties = ResourceGroup(location=rg_location, tags=rg_tags) resource_management_client.resource_groups.create_or_update(resource_group_name, resource_group_properties) else: # Create flag was set to false, this path is not possible through the cli raise WorkspaceException("Resource group was not found.") from azureml._workspace.custom import ml_workspace_create_resources return ml_workspace_create_resources( auth, auth._get_service_client(AzureMachineLearningWorkspaces, subscription_id).workspaces, resource_group_name, workspace_name, location, vnet_location=vnet_location, subscription_id=subscription_id, storage_account=storage_account, key_vault=key_vault, app_insights=app_insights, containerRegistry=containerRegistry, adbWorkspace=adb_workspace, friendly_name=friendly_name, primary_user_assigned_identity=primary_user_assigned_identity, cmk_keyvault=cmk_keyvault, resource_cmk_uri=resource_cmk_uri, hbi_workspace=hbi_workspace, default_cpu_compute_target=default_cpu_compute_target, default_gpu_compute_target=default_gpu_compute_target, private_endpoint_config=private_endpoint_config, private_endpoint_auto_approval=private_endpoint_auto_approval, exist_ok=exist_ok, show_output=show_output, sku=sku, tags=tags, user_assigned_identity_for_cmk_encryption=user_assigned_identity_for_cmk_encryption, system_datastores_auth_mode=system_datastores_auth_mode, is_update_dependent_resources=is_update_dependent_resources)
def create_resource_group(self, resource_management_client, group_name, region, tags): return resource_management_client.resource_groups.create_or_update( group_name, ResourceGroup(location=region, tags=tags))
def temp_vm(): # Generate random value to avoid duplicate resource group hash = random.getrandbits(16) # Defining vars base_name = 'rescue' + str(hash) storage_name = base_name group_name = base_name vm_name = base_name vnet_name = base_name subnet_name = base_name nic_name = base_name os_disk_name = base_name pub_ip_name = base_name computer_name = base_name admin_username='******' admin_password='******' region = orig_vm_location image_publisher = 'Canonical' image_offer = 'UbuntuServer' image_sku = '16.04.0-LTS' image_version = 'latest' # Helper function to create a network interface and vnet def create_network_interface(network_client, region, group_name, interface_name, network_name, subnet_name, ip_name): result = network_client.virtual_networks.create_or_update( group_name, network_name, VirtualNetwork( location=region, address_space=AddressSpace( address_prefixes=[ '10.1.0.0/16', ], ), subnets=[ Subnet( name=subnet_name, address_prefix='10.1.0.0/24', ), ], ), ) print('Creating Virtual Network...') result.wait() # async operation subnet = network_client.subnets.get(group_name, network_name, subnet_name) result = network_client.public_ip_addresses.create_or_update( group_name, ip_name, PublicIPAddress( location=region, public_ip_allocation_method=IPAllocationMethod.dynamic, idle_timeout_in_minutes=4, ), ) print('Creating Subnet...') result.wait() # async operation # Creating Public IP public_ip_address = network_client.public_ip_addresses.get(group_name, ip_name) public_ip_id = public_ip_address.id print('Creating Public IP...') result.wait() # async operation result = network_client.network_interfaces.create_or_update( group_name, interface_name, NetworkInterface( location=region, ip_configurations=[ NetworkInterfaceIPConfiguration( name='default', private_ip_allocation_method=IPAllocationMethod.dynamic, subnet=subnet, public_ip_address=PublicIPAddress( id=public_ip_id, ), ), ], ), ) print('Creating Network Interface...') result.wait() # async operation network_interface = network_client.network_interfaces.get( group_name, interface_name, ) return network_interface.id # 1. Create a resource group print('Creating resource group ' + base_name + '...') result = res_client.resource_groups.create_or_update( group_name, ResourceGroup( location=region, ), ) # 2. Create a storage account print('Creating storage group ' + base_name + '...') result = storage_client.storage_accounts.create( group_name, storage_name, StorageAccountCreateParameters( location=region, sku=Sku(SkuName.premium_lrs), kind=Kind.storage, ), ) result.result() # 3. Create the network interface using a helper function (defined below) nic_id = create_network_interface( network_client, region, group_name, nic_name, vnet_name, subnet_name, pub_ip_name, ) # 4. Create the virtual machine print('Creating temporary VM ' + vm_name + '...') result = compute_client.virtual_machines.create_or_update( group_name, vm_name, VirtualMachine( location=region, os_profile=OSProfile( admin_username=admin_username, admin_password=admin_password, computer_name=computer_name, ), hardware_profile=HardwareProfile( vm_size='Standard_DS1_v2' ), network_profile=NetworkProfile( network_interfaces=[ NetworkInterfaceReference( id=nic_id, ), ], ), storage_profile=StorageProfile( os_disk=OSDisk( caching=CachingTypes.none, create_option=DiskCreateOptionTypes.from_image, name=os_disk_name, vhd=VirtualHardDisk( uri='https://{0}.blob.core.windows.net/vhds/{1}.vhd'.format( storage_name, os_disk_name, ), ), ), image_reference = ImageReference( publisher=image_publisher, offer=image_offer, sku=image_sku, version=image_version, ), ), ), ) result.wait() # async operation # Display the public ip address # You can now connect to the machine using SSH public_ip_address = network_client.public_ip_addresses.get(group_name, pub_ip_name) print('\n' + vm_name + ' has started.') print('VM\'s public IP is {}'.format(public_ip_address.ip_address)) print('SSH Username: '******'SSH Password ' + admin_password) print('ssh ' + admin_username + '@' + public_ip_address.ip_address) # The process of shutting down the VM, deleting it, then removing/attaching OS disk to the temp def disk_attach(): # Delete VM print('Deleting VM and freeing OS disk from ' + orig_vm_name) print('OS Disk Location ' + orig_vm_os_disk) result = compute_client.virtual_machines.delete(sys.argv[2], orig_vm_name) result.wait() # Ensures no lingering lease issues time.sleep(5) # Attach OS disk to temporary VM print('Attaching original OS disk to {0}'.format(vm_name)) result = compute_client.virtual_machines.create_or_update( group_name, vm_name, VirtualMachine( location=orig_vm_location, storage_profile=StorageProfile( data_disks=[DataDisk( lun=0, caching=CachingTypes.none, create_option=DiskCreateOptionTypes.attach, name=orig_vm_name, vhd=VirtualHardDisk( uri=orig_vm_os_disk ) )] ) ) ) result.wait() disk_attach()
def create_resource_group(cmd, rg_name, location): rcf = _resource_client_factory(cmd.cli_ctx) rg_params = ResourceGroup(location=location) return rcf.resource_groups.create_or_update(rg_name, rg_params)
def exec_module(self, **kwargs): for key in self.module_arg_spec.keys() + ['tags']: setattr(self, key, kwargs[key]) results = dict() changed = False rg = None contains_resources = False try: self.log('Fetching resource group {0}'.format(self.name)) rg = self.rm_client.resource_groups.get(self.name) self.check_provisioning_state(rg, self.state) contains_resources = self.resources_exist() results = resource_group_to_dict(rg) if self.state == 'absent': self.log("CHANGED: resource group {0} exists but requested state is 'absent'".format(self.name)) changed = True elif self.state == 'present': update_tags, results['tags'] = self.update_tags(results['tags']) self.log("update tags %s" % update_tags) self.log("new tags: %s" % str(results['tags'])) if update_tags: changed = True if self.location and self.location != results['location']: self.fail("Resource group '{0}' already exists in location '{1}' and cannot be " "moved.".format(self.name, results['location'])) except CloudError: self.log('Resource group {0} does not exist'.format(self.name)) if self.state == 'present': self.log("CHANGED: resource group {0} does not exist but requested state is " "'present'".format(self.name)) changed = True self.results['changed'] = changed self.results['state'] = results self.results['contains_resources'] = contains_resources if self.check_mode: return self.results if changed: if self.state == 'present': if not rg: # Create resource group self.log("Creating resource group {0}".format(self.name)) if not self.location: self.fail("Parameter error: location is required when creating a resource " "group.".format(self.name)) if self.name_exists(): self.fail("Error: a resource group with the name {0} already exists in your subscription." .format(self.name)) params = ResourceGroup( location=self.location, tags=self.tags ) else: # Update resource group params = ResourceGroup( location=results['location'], tags=results['tags'] ) self.results['state'] = self.create_or_update_resource_group(params) elif self.state == 'absent': if contains_resources and not self.force: self.fail("Error removing resource group {0}. Resources exist within the group.".format(self.name)) self.delete_resource_group() return self.results
def get(self, name) -> ResourceGroup: try: with open(self.path / name / "resource_group.json", "r", encoding="utf-8") as file: return ResourceGroup.deserialize(json.load(file)) except FileNotFoundError: raise ResourceNotFoundError("Resource group not found") from None