def __create_vm_state_model_entries(self, infrastructure, num_vms, ec2_secret_key, ec2_access_key, user_id, reservation_id): logging.debug('__create_vm_state_model_entries') logging.debug('num_vms = {0} user_id = {1} reservation_id = {2}'.format(num_vms, user_id, reservation_id)) ids = [] for _ in xrange(num_vms): vm_state = VMStateModel(state=VMStateModel.STATE_CREATING, infra=infrastructure, ec2_access_key=ec2_access_key, ec2_secret_key=ec2_secret_key, user_id=user_id, res_id=reservation_id) vm_state.put() ids.append(vm_state.key().id()) logging.debug('__create_vm_state_model_entries: ids = {0}'.format(ids)) return ids
def prepare_vms(self, parameters): logging.debug('prepare_vms(): parameters={0}'.format(parameters)) queue_head_machine = parameters[self.PARAM_FLEX_QUEUE_HEAD] user_data = self.__get_user_data(parameters['user_id']) if self.PARAM_FLEX_CLOUD_MACHINE_INFO not in parameters \ or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == None \ or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == []: logging.error('Error: No {0} param!'.format( self.PARAM_FLEX_CLOUD_MACHINE_INFO)) # Report Error user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = 'Invalid Parameters' user_data.put() return flex_cloud_machine_info = parameters[ self.PARAM_FLEX_CLOUD_MACHINE_INFO] # Set the user message to "configuring..." user_data.flex_cloud_status = True user_data.flex_cloud_info_msg = 'Flex Cloud configured. Waiting for workers to become available...' user_data.put() # Initialize the VMstateModel db all_accessible = True for machine in flex_cloud_machine_info: if self.agent.check_network_ports(machine['ip'], [22, 443]): state = VMStateModel.STATE_ACCESSIBLE else: state = VMStateModel.STATE_INACCESSIBLE all_accessible = False vm_state = VMStateModel(state=state, infra=self.agent_type, ins_type=FlexConfig.INSTANCE_TYPE, pri_ip=machine['ip'], pub_ip=machine['ip'], username=machine['username'], keyfile=machine['keyfile'], ins_id=self.agent.get_flex_instance_id( machine['ip']), user_id=parameters['user_id'], res_id=self.reservation_id) vm_state.put() if not all_accessible: # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = 'Error: not all workers are accessible' user_data.put() return if queue_head_machine == None or not helper.wait_for_ssh_connection( queue_head_machine['keyfile'], queue_head_machine['ip'], username=queue_head_machine['username']): logging.error( 'Found no viable ssh-able/running queue head machine!') # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = 'Error: Can not connect {0} (queue head) via SSH'.format( queue_head_machine['ip']) user_data.put() return if not self.__prepare_queue_head(queue_head_machine, parameters): logging.error('Error: could not prepare queue head!') # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = 'Error preparing the queue head' user_data.put() return flex_cloud_workers = [] for machine in parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO]: if machine[self.PARAM_IS_QUEUE_HEAD] != True: if helper.wait_for_ssh_connection( machine['keyfile'], machine['ip'], username=machine['username']): flex_cloud_workers.append(machine) else: # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = 'Error: Can not connect to {0} via SSH'.format( machine['ip']) user_data.put() return if len(flex_cloud_workers) > 0: logging.debug( 'Preparing workers: {0}'.format(flex_cloud_workers)) params = { 'infrastructure': AgentTypes.FLEX, self.PARAM_FLEX_CLOUD_MACHINE_INFO: flex_cloud_workers, 'credentials': parameters['credentials'], 'user_id': parameters['user_id'], self.PARAM_FLEX_QUEUE_HEAD: parameters[self.PARAM_FLEX_QUEUE_HEAD], 'reservation_id': parameters['reservation_id'] } self.agent.prepare_instances(params) helper.update_celery_config_with_queue_head_ip( queue_head_ip=queue_head_machine['ip'], agent_type=self.agent_type) self.__configure_celery(params=parameters) # Report Success logging.debug('Flex Cloud Deployed') user_data.flex_cloud_status = True user_data.flex_cloud_info_msg = 'Flex Cloud Deployed' user_data.put() # Force the update of the instance status VMStateModel.synchronize(agent=self.agent, parameters=parameters) return
def prepare_vms(self, parameters): logging.debug("prepare_vms(): parameters={0}".format(parameters)) queue_head_machine = parameters[self.PARAM_FLEX_QUEUE_HEAD] user_data = self.__get_user_data(parameters["user_id"]) if ( self.PARAM_FLEX_CLOUD_MACHINE_INFO not in parameters or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == None or parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] == [] ): logging.error("Error: No {0} param!".format(self.PARAM_FLEX_CLOUD_MACHINE_INFO)) # Report Error user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = "Invalid Parameters" user_data.put() return flex_cloud_machine_info = parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO] # Set the user message to "configuring..." user_data.flex_cloud_status = True user_data.flex_cloud_info_msg = "Flex Cloud configured. Waiting for workers to become available..." user_data.put() # Initialize the VMstateModel db all_accessible = True for machine in flex_cloud_machine_info: if self.agent.check_network_ports(machine["ip"], [22, 443]): state = VMStateModel.STATE_ACCESSIBLE else: state = VMStateModel.STATE_INACCESSIBLE all_accessible = False vm_state = VMStateModel( state=state, infra=self.agent_type, ins_type=FlexConfig.INSTANCE_TYPE, pri_ip=machine["ip"], pub_ip=machine["ip"], username=machine["username"], keyfile=machine["keyfile"], ins_id=self.agent.get_flex_instance_id(machine["ip"]), user_id=parameters["user_id"], res_id=self.reservation_id, ) vm_state.put() if not all_accessible: # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = "Error: not all workers are accessible" user_data.put() return if queue_head_machine == None or not helper.wait_for_ssh_connection( queue_head_machine["keyfile"], queue_head_machine["ip"], username=queue_head_machine["username"] ): logging.error("Found no viable ssh-able/running queue head machine!") # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = "Error: Can not connect {0} (queue head) via SSH".format( queue_head_machine["ip"] ) user_data.put() return if not self.__prepare_queue_head(queue_head_machine, parameters): logging.error("Error: could not prepare queue head!") # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = "Error preparing the queue head" user_data.put() return flex_cloud_workers = [] for machine in parameters[self.PARAM_FLEX_CLOUD_MACHINE_INFO]: if machine[self.PARAM_IS_QUEUE_HEAD] != True: if helper.wait_for_ssh_connection(machine["keyfile"], machine["ip"], username=machine["username"]): flex_cloud_workers.append(machine) else: # Report Failure user_data.flex_cloud_status = False user_data.flex_cloud_info_msg = "Error: Can not connect to {0} via SSH".format(machine["ip"]) user_data.put() return if len(flex_cloud_workers) > 0: logging.debug("Preparing workers: {0}".format(flex_cloud_workers)) params = { "infrastructure": AgentTypes.FLEX, self.PARAM_FLEX_CLOUD_MACHINE_INFO: flex_cloud_workers, "credentials": parameters["credentials"], "user_id": parameters["user_id"], self.PARAM_FLEX_QUEUE_HEAD: parameters[self.PARAM_FLEX_QUEUE_HEAD], "reservation_id": parameters["reservation_id"], } self.agent.prepare_instances(params) helper.update_celery_config_with_queue_head_ip( queue_head_ip=queue_head_machine["ip"], agent_type=self.agent_type ) self.__configure_celery(params=parameters) # Report Success logging.debug("Flex Cloud Deployed") user_data.flex_cloud_status = True user_data.flex_cloud_info_msg = "Flex Cloud Deployed" user_data.put() # Force the update of the instance status VMStateModel.synchronize(agent=self.agent, parameters=parameters) return