def assert_required_parameters(self, parameters, _): """ Checks the given parameters to make sure that they can be used to interact with Google Compute Engine. Args: parameters: A dict that maps the name of each credential to be used in GCE with the value we should use. operation: A BaseAgent.OPERATION that indicates if we wish to add or delete instances. Unused here, as all operations require the same credentials. Raises: AgentConfigurationException: If any of the required credentials are not present, or if the client_secrets parameter refers to a file that is not present on the local filesystem. """ # Make sure the user has set each parameter. for param in self.REQUIRED_CREDENTIALS: if not self.has_parameter(param, parameters): raise AgentConfigurationException('The required parameter, {0}, was' \ ' not specified.'.format(param)) # Next, make sure that either the client_secrets file or the oauth2 # credentials file exists. credentials_file = parameters.get( self.PARAM_SECRETS) or parameters.get(self.PARAM_STORAGE) if not os.path.exists(os.path.expanduser(credentials_file)): raise AgentConfigurationException('Could not find your credentials ' \ 'file at {0}'.format(credentials_file))
def assert_credentials_are_valid(self, parameters): """ Contacts Azure with the given credentials to ensure that they are valid. Gets an access token and a Credentials instance in order to be able to access any resources. Args: parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. Returns: True, if the credentials were valid. A list, of resource group names under the subscription. Raises: AgentConfigurationException: If an error is encountered during authentication. """ credentials = self.open_connection(parameters) subscription_id = parameters[self.PARAM_SUBSCRIBER_ID] try: resource_client = ResourceManagementClient(credentials, subscription_id) resource_groups = resource_client.resource_groups.list() rg_names = [] for rg in resource_groups: rg_names.append(rg.name) return True, rg_names except CloudError as error: raise AgentConfigurationException( "Unable to authenticate using the " "credentials provided. Reason: {}".format(error.message))
def create_storage_account(self, parameters, storage_client): """ Creates a Storage Account under the Resource Group, if it does not already exist. In the case where no resource group is specified, a default storage account is created. Args: parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. credentials: A ServicePrincipalCredentials instance, that can be used to access or create any resources. Raises: AgentConfigurationException: If there was a problem creating or accessing a storage account with the given subscription. """ storage_account = parameters[self.PARAM_STORAGE_ACCOUNT] rg_name = parameters[self.PARAM_RESOURCE_GROUP] try: AppScaleLogger.log( "Creating a new storage account '{0}' under the " "resource group '{1}'.".format(storage_account, rg_name)) result = storage_client.storage_accounts.create( rg_name, storage_account, StorageAccountCreateParameters( sku=Sku(SkuName.standard_lrs), kind=Kind.storage, location=parameters[self.PARAM_ZONE])) # Result is a msrestazure.azure_operation.AzureOperationPoller instance. # wait() insures polling the underlying async operation until it's done. result.wait() except CloudError as error: raise AgentConfigurationException( "Unable to create a storage account " "using the credentials provided: {}".format(error.message))
def get_params_from_yaml(self, keyname): """Searches through the locations.yaml file to build a dict containing the parameters necessary to interact with Amazon EC2. Args: keyname: The name of the SSH keypair that uniquely identifies this AppScale deployment. """ params = { self.PARAM_CREDENTIALS : {}, self.PARAM_GROUP : LocalState.get_group(keyname), self.PARAM_KEYNAME : keyname } zone = LocalState.get_zone(keyname) if zone: params[self.PARAM_REGION] = zone[:-1] else: params[self.PARAM_REGION] = self.DEFAULT_REGION for credential in self.REQUIRED_CREDENTIALS: if os.environ.get(credential): params[self.PARAM_CREDENTIALS][credential] = os.environ[credential] else: raise AgentConfigurationException("no " + credential) return params
def get_cloud_params(self, keyname): """Searches through the locations.json file with key 'infrastructure_info' to build a dict containing the parameters necessary to interact with Amazon EC2. Args: keyname: The name of the SSH keypair that uniquely identifies this AppScale deployment. """ params = { self.PARAM_CREDENTIALS: {}, self.PARAM_GROUP: LocalState.get_group(keyname), self.PARAM_KEYNAME: keyname } zone = LocalState.get_zone(keyname) if zone: params[self.PARAM_REGION] = zone[:-1] else: params[self.PARAM_REGION] = self.DEFAULT_REGION for credential in self.REQUIRED_CREDENTIALS: cred = LocalState.get_infrastructure_option(tag=credential, keyname=keyname) if not cred: raise AgentConfigurationException("no " + credential) params[self.PARAM_CREDENTIALS][credential] = cred return params
def assert_credentials_are_valid(self, parameters): """Contacts GCE to see if the given credentials are valid. Args: parameters: A dict containing the credentials necessary to interact with GCE. Raises: AgentConfigurationException: If an error is encountered during authentication. """ gce_service, credentials = self.open_connection(parameters) try: http = httplib2.Http() auth_http = credentials.authorize(http) request = gce_service.instances().list( project=parameters[self.PARAM_PROJECT], zone=parameters[self.PARAM_ZONE]) response = request.execute(http=auth_http) AppScaleLogger.verbose(str(response), parameters[self.PARAM_VERBOSE]) return True except errors.HttpError as e: error_message = json.loads(e.content)['error']['message'] raise AgentConfigurationException(error_message)
def create_resource_group(self, parameters, credentials): """ Creates a Resource Group for the application using the Service Principal Credentials, if it does not already exist. In the case where no resource group is specified, a default group is created. Args: parameters: A dict, containing all the parameters necessary to authenticate this user with Azure. credentials: A ServicePrincipalCredentials instance, that can be used to access or create any resources. Raises: AgentConfigurationException: If there was a problem creating or accessing a resource group with the given subscription. """ subscription_id = parameters[self.PARAM_SUBSCRIBER_ID] resource_client = ResourceManagementClient(credentials, subscription_id) rg_name = parameters[self.PARAM_RESOURCE_GROUP] tag_name = 'default-tag' if parameters[self.PARAM_TAG]: tag_name = parameters[self.PARAM_TAG] storage_client = StorageManagementClient(credentials, subscription_id) resource_client.providers.register(self.MICROSOFT_STORAGE_RESOURCE) try: # If the resource group does not already exist, create a new one with the # specified storage account. if not parameters[self.PARAM_EXISTING_RG]: AppScaleLogger.log( "Creating a new resource group '{0}' with the tag " "'{1}'.".format(rg_name, tag_name)) resource_client.resource_groups.create_or_update( rg_name, ResourceGroup(location=parameters[self.PARAM_ZONE], tags={'tag': tag_name})) self.create_storage_account(parameters, storage_client) else: # If it already exists, check if the specified storage account exists # under it and if not, create a new account. storage_accounts = storage_client.storage_accounts.\ list_by_resource_group(rg_name) acct_names = [] for account in storage_accounts: acct_names.append(account.name) if parameters[self.PARAM_STORAGE_ACCOUNT] in acct_names: AppScaleLogger.log( "Storage account '{0}' under '{1}' resource group " "already exists. So not creating it again.".format( parameters[self.PARAM_STORAGE_ACCOUNT], rg_name)) else: self.create_storage_account(parameters, storage_client) except CloudError as error: raise AgentConfigurationException( "Unable to create a resource group " "using the credentials provided: {}".format(error.message))
def get_params_from_args(self, args): """ Searches through args to build a dict containing the parameters necessary to interact with Amazon EC2. Args: args: A Namespace containing the arguments that the user has invoked an AppScale Tool with. """ # need to convert this to a dict if it is not already if not isinstance(args, dict): args = vars(args) params = { self.PARAM_CREDENTIALS: {}, self.PARAM_GROUP: args['group'], self.PARAM_IMAGE_ID: args['machine'], self.PARAM_INSTANCE_TYPE: args['instance_type'], self.PARAM_KEYNAME: args['keyname'], self.PARAM_STATIC_IP: args.get(self.PARAM_STATIC_IP), self.PARAM_ZONE: args.get('zone'), 'IS_VERBOSE': args.get('verbose', False), 'autoscale_agent': False } if params[self.PARAM_ZONE]: params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-1] else: params[self.PARAM_REGION] = self.DEFAULT_REGION for credential in self.REQUIRED_CREDENTIALS: if os.environ.get(credential): params[self. PARAM_CREDENTIALS][credential] = os.environ[credential] else: raise AgentConfigurationException("Couldn't find {0} in your " \ "environment. Please set it and run AppScale again." .format(credential)) self.assert_credentials_are_valid(params) if args.get('use_spot_instances') == True: params[self.PARAM_SPOT] = True else: params[self.PARAM_SPOT] = False if params[self.PARAM_SPOT]: if args.get('max_spot_price'): params[self.PARAM_SPOT_PRICE] = args['max_spot_price'] else: params[self.PARAM_SPOT_PRICE] = self.get_optimal_spot_price( self.open_connection(params), params[self.PARAM_INSTANCE_TYPE], params[self.PARAM_ZONE]) return params
def assert_required_parameters(self, parameters, _): """ Checks the given parameters to make sure that they can be used to interact with Google Compute Engine. Args: parameters: A dict that maps the name of each credential to be used in GCE with the value we should use. operation: A BaseAgent.OPERATION that indicates if we wish to add or delete instances. Unused here, as all operations require the same credentials. Raises: AgentConfigurationException: If any of the required credentials are not present, or if the client_secrets parameter refers to a file that is not present on the local filesystem. """ # Make sure the user has set each parameter. for param in self.REQUIRED_CREDENTIALS: if not self.has_parameter(param, parameters): raise AgentConfigurationException('The required parameter, {0}, was' \ ' not specified.'.format(param)) # Next, make sure that either the client_secrets file or the oauth2 # credentials file exists. credentials_file = parameters.get(self.PARAM_SECRETS) or parameters.get( self.PARAM_STORAGE) if not os.path.exists(os.path.expanduser(credentials_file)): raise AgentConfigurationException('Could not find your credentials ' \ 'file at {0}'.format(credentials_file)) # TODO: Remove this warning once service accounts have been fully tested. secrets_location = os.path.expanduser(parameters[self.PARAM_SECRETS]) secrets_type = GCEAgent.get_secrets_type(secrets_location) if (secrets_type == CredentialTypes.SERVICE and not parameters[self.PARAM_TEST]): response = raw_input('It looks like you are using service account ' 'credentials, which are not currently supported for cloud ' 'autoscaling.\nWould you like to continue? (y/N)') if response.lower() not in ['y', 'yes']: raise AgentConfigurationException('User cancelled starting AppScale.')
def assert_required_parameters(self, parameters, operation): required_params = () if operation == BaseAgent.OPERATION_PREPARE: required_params = self.REQUIRED_FLEX_PREPARE_INSTANCES_PARAMS elif operation == BaseAgent.OPERATION_DEREGISTER: required_params = self.REQUIRED_FLEX_DEREGISTER_INSTANCES_PARAMS logging.debug('required_params: {0}'.format(required_params)) for param in required_params: logging.debug('param: {0}'.format(param)) if not utils.has_parameter(param, parameters): raise AgentConfigurationException('no ' + param)
def get_params_from_args(self, args): """ Constructs a dict with only the parameters necessary to interact with Microsoft Azure. Args: args: A Namespace or dict, that maps all of the arguments the user has invoked an AppScale command with their associated value. Returns: A dict, that maps each argument given to the value that was associated with it. Raises: AgentConfigurationException: If unable to authenticate using the credentials provided in the AppScalefile. """ if not isinstance(args, dict): args = vars(args) params = { self.PARAM_APP_ID: args[self.PARAM_APP_ID], self.PARAM_APP_SECRET: args[self.PARAM_APP_SECRET], self.PARAM_IMAGE_ID: args['machine'], self.PARAM_INSTANCE_TYPE: args[self.PARAM_INSTANCE_TYPE], self.PARAM_GROUP: args[self.PARAM_GROUP], self.PARAM_KEYNAME: args[self.PARAM_KEYNAME], self.PARAM_RESOURCE_GROUP: args[self.PARAM_RESOURCE_GROUP], self.PARAM_STORAGE_ACCOUNT: args[self.PARAM_STORAGE_ACCOUNT], self.PARAM_SUBSCRIBER_ID: args[self.PARAM_SUBSCRIBER_ID], self.PARAM_TAG: args[self.PARAM_TAG], self.PARAM_TENANT_ID: args[self.PARAM_TENANT_ID], self.PARAM_TEST: args[self.PARAM_TEST], self.PARAM_VERBOSE: args.get('verbose', False), self.PARAM_ZONE: args[self.PARAM_ZONE] } is_valid, rg_names = self.assert_credentials_are_valid(params) if not is_valid: raise AgentConfigurationException( "Unable to authenticate using the " "credentials provided.") # Check if the resource group passed in exists already, if it does, then # pass an existing group flag so that it is not created again. # In case no resource group is passed, pass a default group. params[self.PARAM_EXISTING_RG] = False if not args[self.PARAM_RESOURCE_GROUP]: params[self.PARAM_RESOURCE_GROUP] = self.DEFAULT_RESOURCE_GROUP if args[self.PARAM_RESOURCE_GROUP] in rg_names: params[self.PARAM_EXISTING_RG] = True if not args[self.PARAM_STORAGE_ACCOUNT]: params[self.PARAM_STORAGE_ACCOUNT] = self.DEFAULT_STORAGE_ACCT return params
def assert_required_parameters(self, parameters, operation): """ Assert that all the parameters required for the EC2 agent are in place. (Also see documentation for the BaseAgent class) Args: parameters: A dictionary of parameters. operation: Operations to be invoked using the above parameters. """ required_params = () if operation == BaseAgent.OPERATION_RUN: required_params = self.REQUIRED_EC2_RUN_INSTANCES_PARAMS elif operation == BaseAgent.OPERATION_TERMINATE: required_params = self.REQUIRED_EC2_TERMINATE_INSTANCES_PARAMS # make sure the user set something for each parameter for param in required_params: if not self.has_parameter(param, parameters): raise AgentConfigurationException('no ' + param) # next, make sure the user actually put in their credentials for credential in self.REQUIRED_EC2_CREDENTIALS: if not self.has_parameter(credential, parameters['credentials']): raise AgentConfigurationException('no ' + credential)
def assert_credentials_are_valid(self, parameters): """Contacts AWS to see if the given access key and secret key represent a valid set of credentials. Args: parameters: A dict containing the user's AWS access key and secret key. Raises: AgentConfigurationException: If the given AWS access key and secret key cannot be used to make requests to AWS. """ conn = self.open_connection(parameters) try: conn.get_all_instances() except EC2ResponseError: raise AgentConfigurationException("We couldn't validate your EC2 " + \ "access key and EC2 secret key. Are your credentials valid?")
def assert_required_parameters(self, parameters, operation): """ Check whether all the parameters required to interact with Azure are present in the provided dict. Args: parameters: A dict containing values necessary to authenticate with the Azure. operation: A str representing the operation for which the parameters should be checked. Raises: AgentConfigurationException: If a required parameter is absent. """ # Make sure that the user has set each parameter. for param in self.REQUIRED_CREDENTIALS: if not self.has_parameter(param, parameters): raise AgentConfigurationException( 'The required parameter, {0}, was not' ' specified.'.format(param))
def assert_required_parameters(self, parameters, operation): """ Assert that all the parameters required for the EC2 agent are in place. (Also see documentation for the BaseAgent class) Args: parameters A dictionary of parameters operation Operations to be invoked using the above parameters """ required_params = () if operation == BaseAgent.OPERATION_PREPARE: required_params = self.REQUIRED_EC2_PREPARE_INSTANCES_PARAMS elif operation == BaseAgent.OPERATION_DEREGISTER: required_params = self.REQUIRED_EC2_DEREGISTER_INSTANCES_PARAMS for param in required_params: if not utils.has_parameter(param, parameters): raise AgentConfigurationException('no ' + param)
def get_params_from_args(self, args): """ Constructs a dict with only the parameters necessary to interact with Google Compute Engine (here, the client_secrets file and the image name). Args: args: A Namespace or dict that maps all of the arguments the user has invoked an AppScale command with their associated value. Returns: A dict containing the location of the client_secrets file and that name of the image to use in GCE. Raises: AgentConfigurationException: If the caller fails to specify a client_secrets file, or if it doesn't exist on the local filesystem. """ if not isinstance(args, dict): args = vars(args) if not args.get('client_secrets') and not args.get('oauth2_storage'): raise AgentConfigurationException("Please specify a client_secrets " + \ "file or a oauth2_storage file in your AppScalefile when running " + \ "over Google Compute Engine.") credentials_file = args.get('client_secrets') or args.get('oauth2_storage') full_credentials = os.path.expanduser(credentials_file) if not os.path.exists(full_credentials): raise AgentConfigurationException("Couldn't find your credentials " + \ "at {0}".format(full_credentials)) if args.get('client_secrets'): destination = LocalState.get_client_secrets_location(args['keyname']) # Make sure the destination's parent directory exists. destination_par = os.path.abspath(os.path.join(destination, os.pardir)) if not os.path.exists(destination_par): os.makedirs(destination_par) shutil.copy(full_credentials, destination) elif args.get('oauth2_storage'): destination = LocalState.get_oauth2_storage_location(args['keyname']) # Make sure the destination's parent directory exists. destination_par = os.path.abspath(os.path.join(destination, os.pardir)) if not os.path.exists(destination_par): os.makedirs(destination_par) shutil.copy(full_credentials, destination) params = { self.PARAM_GROUP : args['group'], self.PARAM_IMAGE_ID : args['machine'], self.PARAM_INSTANCE_TYPE : args[self.PARAM_INSTANCE_TYPE], self.PARAM_KEYNAME : args['keyname'], self.PARAM_PROJECT : args['project'], self.PARAM_STATIC_IP : args.get(self.PARAM_STATIC_IP), self.PARAM_ZONE : args['zone'], self.PARAM_TEST: args['test'], } # A zone in GCE looks like 'us-central2-a', which is in the region # 'us-central2'. Therefore, strip off the last two characters from the zone # to get the region name. if params[self.PARAM_ZONE]: params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-2] else: params[self.PARAM_REGION] = self.DEFAULT_REGION if args.get(self.PARAM_SECRETS): params[self.PARAM_SECRETS] = args.get(self.PARAM_SECRETS) elif args.get(self.PARAM_STORAGE): params[self.PARAM_STORAGE] = args.get(self.PARAM_STORAGE) params[self.PARAM_VERBOSE] = args.get('verbose', False) self.assert_credentials_are_valid(params) return params
def get_params_from_args(self, args): """ Searches through args to build a dict containing the parameters necessary to interact with Amazon EC2. Args: args: A Namespace containing the arguments that the user has invoked an AppScale Tool with. """ # need to convert this to a dict if it is not already if not isinstance(args, dict): args = vars(args) params = { self.PARAM_CREDENTIALS: {}, self.PARAM_GROUP: args['group'], self.PARAM_IMAGE_ID: args['machine'], self.PARAM_INSTANCE_TYPE: args['instance_type'], self.PARAM_KEYNAME: args['keyname'], self.PARAM_STATIC_IP: args.get(self.PARAM_STATIC_IP), self.PARAM_ZONE: args.get('zone'), self.PARAM_VERBOSE: args.get('verbose', False), self.PARAM_AUTOSCALE_AGENT: False } if params[self.PARAM_ZONE]: params[self.PARAM_REGION] = params[self.PARAM_ZONE][:-1] else: params[self.PARAM_REGION] = self.DEFAULT_REGION for credential in self.REQUIRED_CREDENTIALS: if args.get(credential): params[self.PARAM_CREDENTIALS][credential] = args[credential] else: raise AgentConfigurationException("Couldn't find {0} in your " \ "environment. Please set it and run AppScale again." .format(credential)) self.assert_credentials_are_valid(params) if args.get('use_spot_instances') == True: params[self.PARAM_SPOT] = True else: params[self.PARAM_SPOT] = False if params[self.PARAM_SPOT]: if args.get('max_spot_price'): params[self.PARAM_SPOT_PRICE] = args['max_spot_price'] else: params[self.PARAM_SPOT_PRICE] = self.get_optimal_spot_price( self.open_connection(params), params[self.PARAM_INSTANCE_TYPE], params[self.PARAM_ZONE]) # If VPC id and Subnet id are not set assume classic networking should be # used. vpc_id = args.get(self.PARAM_VPC_ID) subnet_id = args.get(self.PARAM_SUBNET_ID) if not vpc_id and not subnet_id: AppScaleLogger.log( 'Using Classic Networking since subnet and vpc were ' 'not specified.') # All further checks are for VPC Networking. elif (vpc_id or subnet_id) and not (vpc_id and subnet_id): raise AgentConfigurationException( 'Both VPC id and Subnet id must be ' 'specified to use VPC Networking.') else: # VPC must exist. vpc_conn = self.open_vpc_connection(params) params[self.PARAM_VPC_ID] = args[self.PARAM_VPC_ID] try: vpc_conn.get_all_vpcs(params[self.PARAM_VPC_ID]) except EC2ResponseError as e: raise AgentConfigurationException( 'Error looking for vpc: {}'.format(e.message)) # Subnet must exist. all_subnets = vpc_conn.get_all_subnets( filters={'vpcId': params[self.PARAM_VPC_ID]}) params[self.PARAM_SUBNET_ID] = args[self.PARAM_SUBNET_ID] if not any(subnet.id == params[self.PARAM_SUBNET_ID] for subnet in all_subnets): raise AgentConfigurationException( 'Specified subnet {} does not exist ' 'in vpc {}!'.format(params[self.PARAM_SUBNET_ID], params[self.PARAM_VPC_ID])) return params