def stop_virtual_machine(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot) now_status = adapter.get_virtual_machine_instance_status( ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name) if now_status is None: self.log.error( "azure virtual environment %d stop vm failed: cannot get status of vm %r" % (sctx.current_job_index, ctx.virtual_machine_name)) self.__on_stop_virtual_machine_failed(sctx) elif now_status != AVMStatus.STOPPED_DEALLOCATED: try: req = adapter.stop_virtual_machine( ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name, AVMStatus.STOPPED_DEALLOCATED) except Exception as e: self.log.error( "azure virtual environment %d stop vm failed: %r" % (sctx.current_job_index, str(e.message))) self.__on_stop_virtual_machine_failed(sctx) return False ctx.request_id = req.request_id self.__wait_for_stop_virtual_machine(sctx) else: self.__stop_virtual_machine_done(sctx)
def __config_virtual_machine(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] ctx.vm_need_config = False adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) if len(network_config.input_endpoints.input_endpoints) == 0: # don't need to config, skip self.__setup_virtual_machine_done(sctx) return try: req = adapter.update_virtual_machine_network_config( ctx.cloud_service_name, ctx.deployment_name, ctx.virtual_machine_name, network_config) except Exception as e: self.log.error( "azure virtual environment %d error while config network: %r" % (sctx.current_job_index, e.message)) self.__on_setup_failed(sctx) return ctx.request_id = req.request_id self.__wait_for_config_virtual_machine(sctx)
def __setup_virtual_machine_without_deployment_existed(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if not ctx.is_vm_image: network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) else: network_config = None try: req = adapter.create_virtual_machine_deployment( ctx.cloud_service_name, ctx.deployment_name, ctx.deployment_slot, ctx.virtual_machine_label, ctx.virtual_machine_name, ctx.system_config, ctx.os_virtual_hard_disk, network_config, role_size=ctx.virtual_machine_size, vm_image_name=ctx.image_name if ctx.is_vm_image else None) except Exception as e: self.log.error( "azure virtual environment %d create virtual machine %r failed: %r" % (sctx.current_job_index, ctx.virtual_machine_name, str(e))) self.__on_setup_failed(sctx) return # wait for add virtual machine to finish ctx.request_id = req.request_id ctx.vm_need_config = True if ctx.is_vm_image else False self.__wait_for_create_virtual_machine_deployment(sctx)
def wait_for_virtual_machine_ready(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) status = adapter.get_virtual_machine_instance_status( ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name) if not status: self.log.error( "azure virtual environment %d error occured while waiting for virtual machine ready" % sctx.current_job_index) self.__on_setup_failed(sctx) return self.log.debug( "waiting for virtual machine ready, vm: %s, status: %s" % (ctx.virtual_machine_name, str(status))) if status == AVMStatus.READY_ROLE: if ctx.vm_need_config: ctx.vm_need_config = False self.__config_virtual_machine(sctx) else: self.__setup_virtual_machine_done(sctx) else: self.__wait_for_virtual_machine_ready(sctx)
def __config_virtual_machine(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] ctx.vm_need_config = False adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) if len(network_config.input_endpoints.input_endpoints) == 0: # don't need to config, skip self.__setup_virtual_machine_done(sctx) return try: req = adapter.update_virtual_machine_network_config( ctx.cloud_service_name, ctx.deployment_name, ctx.virtual_machine_name, network_config) except Exception as e: self.log.error( "azure virtual environment %d error while config network: %r" % (sctx.current_job_index, e.message)) self.__on_setup_failed(sctx) return ctx.request_id = req.request_id self.__wait_for_config_virtual_machine(sctx)
def wait_for_virtual_machine_ready(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) status = adapter.get_virtual_machine_instance_status( ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name) if not status: self.log.error( "azure virtual environment %d error occured while waiting for virtual machine ready" % sctx.current_job_index) self.__on_setup_failed(sctx) return self.log.debug( "waiting for virtual machine ready, vm: %s, status: %s" % (ctx.virtual_machine_name, str(status))) if status == AVMStatus.READY_ROLE: if ctx.vm_need_config: ctx.vm_need_config = False self.__config_virtual_machine(sctx) else: self.__setup_virtual_machine_done(sctx) else: self.__wait_for_virtual_machine_ready(sctx)
def __setup_virtual_machine_done(self, sctx): self.log.debug("azure virtual environment %d vm setup done" % sctx.current_job_index) ctx = sctx.job_ctxs[sctx.current_job_index] # update the status of virtual environment ve = self.db.find_first_object_by(VirtualEnvironment, id=ctx.virtual_environment_id) adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if ve: ve.status = VEStatus.RUNNING public_ip, port = adapter.get_virtual_machine_public_endpoint( ctx.cloud_service_name, ctx.deployment_name, ctx.virtual_machine_name, ctx.endpoint_name) if not public_ip: self.log.warn("unable to find public ip for vm %s, set guacamole failed" % ctx.virtual_machine_name) else: remote_para = get_remote_parameters( ctx.raw_system_config, ctx.remote, ctx.virtual_machine_name, public_ip, port) ve.remote_paras = json.dumps(remote_para) self.db.commit() self.expr_manager.check_expr_status(ve.experiment) self.log.debug("azure virtual environment %d vm success callback done, step to next" % sctx.current_job_index) # step to config next unit sctx.current_job_index += 1 self.__schedule_setup(sctx)
def stop_virtual_machine(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot) now_status = adapter.get_virtual_machine_instance_status( ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name) if now_status is None: self.log.error( "azure virtual environment %d stop vm failed: cannot get status of vm %r" % (sctx.current_job_index, ctx.virtual_machine_name)) self.__on_stop_virtual_machine_failed(sctx) elif now_status != AVMStatus.STOPPED_DEALLOCATED: try: req = adapter.stop_virtual_machine( ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name, AVMStatus.STOPPED_DEALLOCATED) except Exception as e: self.log.error( "azure virtual environment %d stop vm failed: %r" % (sctx.current_job_index, str(e.message))) self.__on_stop_virtual_machine_failed(sctx) return False ctx.request_id = req.request_id self.__wait_for_stop_virtual_machine(sctx) else: self.__stop_virtual_machine_done(sctx)
def setup_virtual_machine(self, sctx): # get context from super context ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if adapter.deployment_exists(ctx.cloud_service_name, ctx.deployment_slot): # TODO: need to store the deployment info into db? self.__setup_virtual_machine_with_deployment_existed(sctx) else: self.__setup_virtual_machine_without_deployment_existed(sctx)
def __check_vm_operation_status(self, sctx, on_success, on_failed, on_continue): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) res = adapter.get_operation_status(ctx.request_id) if res.status == ASYNC_OP_RESULT.SUCCEEDED: on_success(sctx) elif res.error: on_failed(sctx) else: on_continue(sctx)
def __setup_virtual_machine_done(self, sctx): self.log.debug("azure virtual environment %d vm setup done" % sctx.current_job_index) ctx = sctx.job_ctxs[sctx.current_job_index] # update the status of virtual environment expr = Experiment.objects(id=ctx.experiment_id).first() ve = expr.virtual_environments[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) ve.status = VEStatus.RUNNING expr.save() azure_resource = AzureVirtualMachine(name=ctx.virtual_machine_name, label=ctx.virtual_machine_label, dns="%s.chinacloudapp.cn" % ctx.cloud_service_name, end_points=[]) # todo record AzureDeployment, AzureCloudService and so on in db for roll back vm_role = adapter.get_virtual_machine_role(ctx.cloud_service_name, ctx.deployment_name, ctx.virtual_machine_name) if (not vm_role) or (not vm_role.instance_endpoints): self.log.warn( "unable to find vm %s, cannot update virtual env config like guacamole" % ctx.virtual_machine_name) else: for endpoint in vm_role.instance_endpoints: azure_resource.public_ip = endpoint.vip if endpoint.name == ctx.remote_endpoint_name: # endpoint for remote desktop ve.remote_provider = VERemoteProvider.Guacamole ve.remote_paras = get_remote_parameters( ctx.raw_system_config, ctx.remote, ctx.virtual_machine_name, endpoint.vip, endpoint.public_port) else: try: aep = self.__get_persistable_endpoint(endpoint, ctx.raw_network_config) azure_resource.end_points.append(aep) except Exception as e: self.log.error(e) ve.azure_resource = azure_resource azure_resource.save() expr.save() self.expr_manager.check_expr_status(expr) self.log.debug("azure virtual environment %d vm success callback done, step to next" % sctx.current_job_index) # step to config next unit sctx.current_job_index += 1 self.__schedule_setup(sctx)
def setup_virtual_machine(self, sctx): # get context from super context ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if adapter.deployment_exists(ctx.cloud_service_name, ctx.deployment_slot): # TODO: need to store the deployment info into db? self.__setup_virtual_machine_with_deployment_existed(sctx) else: self.__setup_virtual_machine_without_deployment_existed(sctx)
def __check_vm_operation_status(self, sctx, on_success, on_failed, on_continue): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) res = adapter.get_operation_status(ctx.request_id) if res.status == ASYNC_OP_RESULT.SUCCEEDED: on_success(sctx) elif res.error: on_failed(sctx) else: on_continue(sctx)
def __setup_virtual_machine_with_deployment_existed(self, sctx): # get context from super context ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot) if not ctx.is_vm_image: network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) else: network_config = None # add virtual machine to deployment try: # create if vm is not created if not adapter.virtual_machine_exists(ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name): req = adapter.add_virtual_machine( ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name, ctx.system_config, ctx.os_virtual_hard_disk, network_config=network_config, provision_guest_agent=True, resource_extension_references=ctx.resource_extension_references, role_size=ctx.virtual_machine_size, vm_image_name=ctx.image_name if ctx.is_vm_image else None) # NOTE: if the vm is created, we think is well configured # else: # if vm is created, then we need to config the vm # ctx.vm_need_config = True # self.__wait_for_virtual_machine_ready(sctx) # return else: self.__setup_virtual_machine_done(sctx) return except Exception as e: self.log.error( "azure virtual environment %d create virtual machine %r failed: %r" % (sctx.current_job_index, ctx.virtual_machine_name, str(e))) self.__on_setup_failed(sctx) return # wait for add virtual machine to finish ctx.request_id = req.request_id ctx.vm_need_config = True if ctx.is_vm_image else False self.__wait_for_add_virtual_machine(sctx)
def __setup_rollback(self, record, sctx): # TODO: the rollback process should use a async way same # as previous setup process, but for convinence, # we do it in a sync way for rec in record: if rec.type == REMOTE_CREATED_RECORD.TYPE_CLOUD_SERVICE: adapter = CloudServiceAdapter(sctx.subscription_id, sctx.pem_url, host=sctx.management_host) adapter.delete_cloud_service(rec.name, complete=True) elif rec.type == REMOTE_CREATED_RECORD.TYPE_STORAGE_ACCOUNT: adapter = StorageAccountAdapter(sctx.subscription_id, sctx.pem_url, host=sctx.management_host) adapter.delete_storage_account(rec.name) elif (rec.type == REMOTE_CREATED_RECORD.TYPE_ADD_VIRTUAL_MACHINE or rec.type == REMOTE_CREATED_RECORD. TYPE_CREATE_VIRTUAL_MACHINE_DEPLOYMENT): adapter = VirtualMachineAdapter(sctx.subscription_id, sctx.pem_url, host=sctx.management_host) if rec.type == REMOTE_CREATED_RECORD.TYPE_ADD_VIRTUAL_MACHINE: adapter.delete_virtual_machine(rec.cloud_service_name, rec.deployment_name, rec.virtual_machine_name, True) else: adapter.delete_deployment(rec.cloud_service_name, rec.deployment_name, True) else: self.log.warn("unknown record type: %s" % rec.type)
def wait_for_deployment_ready(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) props = adapter.get_deployment_by_slot(ctx.cloud_service_name, ctx.deployment_slot) if not props: self.log.error( "azure virtual environment %d error occured while waiting for deployment ready" % sctx.current_job_index) self.__on_setup_failed(sctx) return if props.status == ADStatus.RUNNING: self.__wait_for_virtual_machine_ready(sctx) else: self.__wait_for_create_virtual_machine_deployment(sctx)
def wait_for_deployment_ready(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) props = adapter.get_deployment_by_slot(ctx.cloud_service_name, ctx.deployment_slot) if not props: self.log.error( "azure virtual environment %d error occured while waiting for deployment ready" % sctx.current_job_index) self.__on_setup_failed(sctx) return if props.status == ADStatus.RUNNING: self.__wait_for_virtual_machine_ready(sctx) else: self.__wait_for_create_virtual_machine_deployment(sctx)
def __setup_virtual_machine_done(self, sctx): self.log.debug("azure virtual environment %d vm setup done" % sctx.current_job_index) ctx = sctx.job_ctxs[sctx.current_job_index] # update the status of virtual environment ve = self.db.find_first_object_by(VirtualEnvironment, id=ctx.virtual_environment_id) adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if ve: ve.status = VEStatus.RUNNING public_ip, port = adapter.get_virtual_machine_public_endpoint( ctx.cloud_service_name, ctx.deployment_name, ctx.virtual_machine_name, ctx.endpoint_name) if not public_ip: self.log.warn( "unable to find public ip for vm %s, set guacamole failed" % ctx.virtual_machine_name) else: remote_para = get_remote_parameters(ctx.raw_system_config, ctx.remote, ctx.virtual_machine_name, public_ip, port) ve.remote_paras = json.dumps(remote_para) self.db.commit() self.expr_manager.check_expr_status(ve.experiment) self.log.debug( "azure virtual environment %d vm success callback done, step to next" % sctx.current_job_index) # step to config next unit sctx.current_job_index += 1 self.__schedule_setup(sctx)
def __setup_virtual_machine_without_deployment_existed(self, sctx): ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) if not ctx.is_vm_image: network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) else: network_config = None try: req = adapter.create_virtual_machine_deployment( ctx.cloud_service_name, ctx.deployment_name, ctx.deployment_slot, ctx.virtual_machine_label, ctx.virtual_machine_name, ctx.system_config, ctx.os_virtual_hard_disk, network_config, role_size=ctx.virtual_machine_size, vm_image_name=ctx.image_name if ctx.is_vm_image else None) except Exception as e: self.log.error( "azure virtual environment %d create virtual machine %r failed: %r" % (sctx.current_job_index, ctx.virtual_machine_name, str(e))) self.__on_setup_failed(sctx) return # wait for add virtual machine to finish ctx.request_id = req.request_id ctx.vm_need_config = True if ctx.is_vm_image else False self.__wait_for_create_virtual_machine_deployment(sctx)
def __setup_virtual_machine_with_deployment_existed(self, sctx): # get context from super context ctx = sctx.job_ctxs[sctx.current_job_index] adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host) deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot) if not ctx.is_vm_image: network_config = get_network_config( ctx.raw_network_config, adapter.get_assigned_endpoints(ctx.cloud_service_name)) else: network_config = None # add virtual machine to deployment try: # create if vm is not created if not adapter.virtual_machine_exists(ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name): req = adapter.add_virtual_machine( ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name, ctx.system_config, ctx.os_virtual_hard_disk, network_config=network_config, role_size=ctx.virtual_machine_size, vm_image_name=ctx.image_name if ctx.is_vm_image else None) # NOTE: if the vm is created, we think is well configured # else: # if vm is created, then we need to config the vm # ctx.vm_need_config = True # self.__wait_for_virtual_machine_ready(sctx) # return else: self.__setup_virtual_machine_done(sctx) return except Exception as e: self.log.error( "azure virtual environment %d create virtual machine %r failed: %r" % (sctx.current_job_index, ctx.virtual_machine_name, str(e))) self.__on_setup_failed(sctx) return # wait for add virtual machine to finish ctx.request_id = req.request_id ctx.vm_need_config = True if ctx.is_vm_image else False self.__wait_for_add_virtual_machine(sctx)