Пример #1
0
    def stop_virtual_machine(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        deployment_name = adapter.get_deployment_name(ctx.cloud_service_name,
                                                      ctx.deployment_slot)
        now_status = adapter.get_virtual_machine_instance_status(
            ctx.cloud_service_name, ctx.deployment_slot,
            ctx.virtual_machine_name)

        if now_status is None:
            self.log.error(
                "azure virtual environment %d stop vm failed: cannot get status of vm %r"
                % (sctx.current_job_index, ctx.virtual_machine_name))
            self.__on_stop_virtual_machine_failed(sctx)
        elif now_status != AVMStatus.STOPPED_DEALLOCATED:
            try:
                req = adapter.stop_virtual_machine(
                    ctx.cloud_service_name, deployment_name,
                    ctx.virtual_machine_name, AVMStatus.STOPPED_DEALLOCATED)
            except Exception as e:
                self.log.error(
                    "azure virtual environment %d stop vm failed: %r" %
                    (sctx.current_job_index, str(e.message)))
                self.__on_stop_virtual_machine_failed(sctx)
                return False

            ctx.request_id = req.request_id
            self.__wait_for_stop_virtual_machine(sctx)
        else:
            self.__stop_virtual_machine_done(sctx)
Пример #2
0
    def __config_virtual_machine(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        ctx.vm_need_config = False
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        network_config = get_network_config(
            ctx.raw_network_config,
            adapter.get_assigned_endpoints(ctx.cloud_service_name))

        if len(network_config.input_endpoints.input_endpoints) == 0:
            # don't need to config, skip
            self.__setup_virtual_machine_done(sctx)
            return

        try:
            req = adapter.update_virtual_machine_network_config(
                ctx.cloud_service_name,
                ctx.deployment_name,
                ctx.virtual_machine_name,
                network_config)
        except Exception as e:
            self.log.error(
                "azure virtual environment %d error while config network: %r" %
                (sctx.current_job_index, e.message))
            self.__on_setup_failed(sctx)
            return

        ctx.request_id = req.request_id
        self.__wait_for_config_virtual_machine(sctx)
Пример #3
0
    def __setup_virtual_machine_without_deployment_existed(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        if not ctx.is_vm_image:
            network_config = get_network_config(
                ctx.raw_network_config,
                adapter.get_assigned_endpoints(ctx.cloud_service_name))
        else:
            network_config = None

        try:
            req = adapter.create_virtual_machine_deployment(
                ctx.cloud_service_name,
                ctx.deployment_name,
                ctx.deployment_slot,
                ctx.virtual_machine_label,
                ctx.virtual_machine_name,
                ctx.system_config,
                ctx.os_virtual_hard_disk,
                network_config,
                role_size=ctx.virtual_machine_size,
                vm_image_name=ctx.image_name if ctx.is_vm_image else None)
        except Exception as e:
            self.log.error(
                "azure virtual environment %d create virtual machine %r failed: %r"
                % (sctx.current_job_index, ctx.virtual_machine_name, str(e)))
            self.__on_setup_failed(sctx)
            return

        # wait for add virtual machine to finish
        ctx.request_id = req.request_id
        ctx.vm_need_config = True if ctx.is_vm_image else False
        self.__wait_for_create_virtual_machine_deployment(sctx)
Пример #4
0
    def wait_for_virtual_machine_ready(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        status = adapter.get_virtual_machine_instance_status(
            ctx.cloud_service_name, ctx.deployment_slot,
            ctx.virtual_machine_name)

        if not status:
            self.log.error(
                "azure virtual environment %d error occured while waiting for virtual machine ready"
                % sctx.current_job_index)
            self.__on_setup_failed(sctx)
            return

        self.log.debug(
            "waiting for virtual machine ready, vm: %s, status: %s" %
            (ctx.virtual_machine_name, str(status)))
        if status == AVMStatus.READY_ROLE:
            if ctx.vm_need_config:
                ctx.vm_need_config = False
                self.__config_virtual_machine(sctx)
            else:
                self.__setup_virtual_machine_done(sctx)
        else:
            self.__wait_for_virtual_machine_ready(sctx)
Пример #5
0
    def __config_virtual_machine(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        ctx.vm_need_config = False
        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        network_config = get_network_config(
            ctx.raw_network_config,
            adapter.get_assigned_endpoints(ctx.cloud_service_name))

        if len(network_config.input_endpoints.input_endpoints) == 0:
            # don't need to config, skip
            self.__setup_virtual_machine_done(sctx)
            return

        try:
            req = adapter.update_virtual_machine_network_config(
                ctx.cloud_service_name, ctx.deployment_name,
                ctx.virtual_machine_name, network_config)
        except Exception as e:
            self.log.error(
                "azure virtual environment %d error while config network: %r" %
                (sctx.current_job_index, e.message))
            self.__on_setup_failed(sctx)
            return

        ctx.request_id = req.request_id
        self.__wait_for_config_virtual_machine(sctx)
Пример #6
0
    def wait_for_virtual_machine_ready(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        status = adapter.get_virtual_machine_instance_status(
            ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name)

        if not status:
            self.log.error(
                "azure virtual environment %d error occured while waiting for virtual machine ready"
                % sctx.current_job_index)
            self.__on_setup_failed(sctx)
            return

        self.log.debug(
            "waiting for virtual machine ready, vm: %s, status: %s" %
            (ctx.virtual_machine_name, str(status)))
        if status == AVMStatus.READY_ROLE:
            if ctx.vm_need_config:
                ctx.vm_need_config = False
                self.__config_virtual_machine(sctx)
            else:
                self.__setup_virtual_machine_done(sctx)
        else:
            self.__wait_for_virtual_machine_ready(sctx)
Пример #7
0
    def __setup_virtual_machine_done(self, sctx):
        self.log.debug("azure virtual environment %d vm setup done" % sctx.current_job_index)
        ctx = sctx.job_ctxs[sctx.current_job_index]

        # update the status of virtual environment
        ve = self.db.find_first_object_by(VirtualEnvironment, id=ctx.virtual_environment_id)

        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)
        if ve:
            ve.status = VEStatus.RUNNING

            public_ip, port = adapter.get_virtual_machine_public_endpoint(
                ctx.cloud_service_name,
                ctx.deployment_name,
                ctx.virtual_machine_name,
                ctx.endpoint_name)

            if not public_ip:
                self.log.warn("unable to find public ip for vm %s, set guacamole failed" % ctx.virtual_machine_name)
            else:
                remote_para = get_remote_parameters(
                    ctx.raw_system_config,
                    ctx.remote,
                    ctx.virtual_machine_name,
                    public_ip, port)

                ve.remote_paras = json.dumps(remote_para)

            self.db.commit()
            self.expr_manager.check_expr_status(ve.experiment)

        self.log.debug("azure virtual environment %d vm success callback done, step to next" % sctx.current_job_index)
        # step to config next unit
        sctx.current_job_index += 1
        self.__schedule_setup(sctx)
Пример #8
0
    def stop_virtual_machine(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot)
        now_status = adapter.get_virtual_machine_instance_status(
            ctx.cloud_service_name, ctx.deployment_slot, ctx.virtual_machine_name)

        if now_status is None:
            self.log.error(
                "azure virtual environment %d stop vm failed: cannot get status of vm %r" %
                (sctx.current_job_index, ctx.virtual_machine_name))
            self.__on_stop_virtual_machine_failed(sctx)
        elif now_status != AVMStatus.STOPPED_DEALLOCATED:
            try:
                req = adapter.stop_virtual_machine(
                    ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name, AVMStatus.STOPPED_DEALLOCATED)
            except Exception as e:
                self.log.error(
                    "azure virtual environment %d stop vm failed: %r" %
                    (sctx.current_job_index, str(e.message)))
                self.__on_stop_virtual_machine_failed(sctx)
                return False

            ctx.request_id = req.request_id
            self.__wait_for_stop_virtual_machine(sctx)
        else:
            self.__stop_virtual_machine_done(sctx)
Пример #9
0
    def setup_virtual_machine(self, sctx):
        # get context from super context
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        if adapter.deployment_exists(ctx.cloud_service_name, ctx.deployment_slot):
            # TODO: need to store the deployment info into db?
            self.__setup_virtual_machine_with_deployment_existed(sctx)
        else:
            self.__setup_virtual_machine_without_deployment_existed(sctx)
Пример #10
0
    def __check_vm_operation_status(self, sctx, on_success, on_failed, on_continue):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        res = adapter.get_operation_status(ctx.request_id)

        if res.status == ASYNC_OP_RESULT.SUCCEEDED:
            on_success(sctx)
        elif res.error:
            on_failed(sctx)
        else:
            on_continue(sctx)
Пример #11
0
    def __setup_virtual_machine_done(self, sctx):
        self.log.debug("azure virtual environment %d vm setup done" % sctx.current_job_index)
        ctx = sctx.job_ctxs[sctx.current_job_index]

        # update the status of virtual environment
        expr = Experiment.objects(id=ctx.experiment_id).first()
        ve = expr.virtual_environments[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        ve.status = VEStatus.RUNNING
        expr.save()

        azure_resource = AzureVirtualMachine(name=ctx.virtual_machine_name,
                                             label=ctx.virtual_machine_label,
                                             dns="%s.chinacloudapp.cn" % ctx.cloud_service_name,
                                             end_points=[])
        # todo record AzureDeployment, AzureCloudService and so on in db for roll back

        vm_role = adapter.get_virtual_machine_role(ctx.cloud_service_name,
                                                   ctx.deployment_name,
                                                   ctx.virtual_machine_name)

        if (not vm_role) or (not vm_role.instance_endpoints):
            self.log.warn(
                "unable to find vm %s, cannot update virtual env config like guacamole" % ctx.virtual_machine_name)
        else:
            for endpoint in vm_role.instance_endpoints:
                azure_resource.public_ip = endpoint.vip
                if endpoint.name == ctx.remote_endpoint_name:  # endpoint for remote desktop
                    ve.remote_provider = VERemoteProvider.Guacamole
                    ve.remote_paras = get_remote_parameters(
                        ctx.raw_system_config,
                        ctx.remote,
                        ctx.virtual_machine_name,
                        endpoint.vip,
                        endpoint.public_port)
                else:
                    try:
                        aep = self.__get_persistable_endpoint(endpoint, ctx.raw_network_config)
                        azure_resource.end_points.append(aep)
                    except Exception as e:
                        self.log.error(e)

        ve.azure_resource = azure_resource
        azure_resource.save()
        expr.save()
        self.expr_manager.check_expr_status(expr)

        self.log.debug("azure virtual environment %d vm success callback done, step to next" % sctx.current_job_index)
        # step to config next unit
        sctx.current_job_index += 1
        self.__schedule_setup(sctx)
Пример #12
0
    def setup_virtual_machine(self, sctx):
        # get context from super context
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        if adapter.deployment_exists(ctx.cloud_service_name,
                                     ctx.deployment_slot):
            # TODO: need to store the deployment info into db?
            self.__setup_virtual_machine_with_deployment_existed(sctx)
        else:
            self.__setup_virtual_machine_without_deployment_existed(sctx)
Пример #13
0
    def __check_vm_operation_status(self, sctx, on_success, on_failed,
                                    on_continue):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        res = adapter.get_operation_status(ctx.request_id)

        if res.status == ASYNC_OP_RESULT.SUCCEEDED:
            on_success(sctx)
        elif res.error:
            on_failed(sctx)
        else:
            on_continue(sctx)
Пример #14
0
    def __setup_virtual_machine_with_deployment_existed(self, sctx):
        # get context from super context
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        deployment_name = adapter.get_deployment_name(ctx.cloud_service_name, ctx.deployment_slot)
        if not ctx.is_vm_image:
            network_config = get_network_config(
                ctx.raw_network_config,
                adapter.get_assigned_endpoints(ctx.cloud_service_name))
        else:
            network_config = None

        # add virtual machine to deployment
        try:
            # create if vm is not created
            if not adapter.virtual_machine_exists(ctx.cloud_service_name, deployment_name, ctx.virtual_machine_name):
                req = adapter.add_virtual_machine(
                    ctx.cloud_service_name,
                    deployment_name,
                    ctx.virtual_machine_name,
                    ctx.system_config,
                    ctx.os_virtual_hard_disk,
                    network_config=network_config,
                    provision_guest_agent=True,
                    resource_extension_references=ctx.resource_extension_references,
                    role_size=ctx.virtual_machine_size,
                    vm_image_name=ctx.image_name if ctx.is_vm_image else None)
            # NOTE: if the vm is created, we think is well configured
            # else:  # if vm is created, then we need to config the vm
            #    ctx.vm_need_config = True
            #    self.__wait_for_virtual_machine_ready(sctx)
            #    return
            else:
                self.__setup_virtual_machine_done(sctx)
                return
        except Exception as e:
            self.log.error(
                "azure virtual environment %d create virtual machine %r failed: %r"
                % (sctx.current_job_index, ctx.virtual_machine_name, str(e)))
            self.__on_setup_failed(sctx)
            return

        # wait for add virtual machine to finish
        ctx.request_id = req.request_id
        ctx.vm_need_config = True if ctx.is_vm_image else False
        self.__wait_for_add_virtual_machine(sctx)
Пример #15
0
    def __setup_rollback(self, record, sctx):
        # TODO: the rollback process should use a async way same
        #       as previous setup process, but for convinence,
        #       we do it in a sync way
        for rec in record:
            if rec.type == REMOTE_CREATED_RECORD.TYPE_CLOUD_SERVICE:
                adapter = CloudServiceAdapter(sctx.subscription_id,
                                              sctx.pem_url,
                                              host=sctx.management_host)
                adapter.delete_cloud_service(rec.name, complete=True)
            elif rec.type == REMOTE_CREATED_RECORD.TYPE_STORAGE_ACCOUNT:
                adapter = StorageAccountAdapter(sctx.subscription_id,
                                                sctx.pem_url,
                                                host=sctx.management_host)
                adapter.delete_storage_account(rec.name)
            elif (rec.type == REMOTE_CREATED_RECORD.TYPE_ADD_VIRTUAL_MACHINE
                  or rec.type == REMOTE_CREATED_RECORD.
                  TYPE_CREATE_VIRTUAL_MACHINE_DEPLOYMENT):
                adapter = VirtualMachineAdapter(sctx.subscription_id,
                                                sctx.pem_url,
                                                host=sctx.management_host)

                if rec.type == REMOTE_CREATED_RECORD.TYPE_ADD_VIRTUAL_MACHINE:
                    adapter.delete_virtual_machine(rec.cloud_service_name,
                                                   rec.deployment_name,
                                                   rec.virtual_machine_name,
                                                   True)
                else:
                    adapter.delete_deployment(rec.cloud_service_name,
                                              rec.deployment_name, True)
            else:
                self.log.warn("unknown record type: %s" % rec.type)
Пример #16
0
    def wait_for_deployment_ready(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id, ctx.pem_url, host=ctx.management_host)

        props = adapter.get_deployment_by_slot(ctx.cloud_service_name, ctx.deployment_slot)

        if not props:
            self.log.error(
                "azure virtual environment %d error occured while waiting for deployment ready"
                % sctx.current_job_index)
            self.__on_setup_failed(sctx)
            return

        if props.status == ADStatus.RUNNING:
            self.__wait_for_virtual_machine_ready(sctx)
        else:
            self.__wait_for_create_virtual_machine_deployment(sctx)
Пример #17
0
    def wait_for_deployment_ready(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]
        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        props = adapter.get_deployment_by_slot(ctx.cloud_service_name,
                                               ctx.deployment_slot)

        if not props:
            self.log.error(
                "azure virtual environment %d error occured while waiting for deployment ready"
                % sctx.current_job_index)
            self.__on_setup_failed(sctx)
            return

        if props.status == ADStatus.RUNNING:
            self.__wait_for_virtual_machine_ready(sctx)
        else:
            self.__wait_for_create_virtual_machine_deployment(sctx)
Пример #18
0
    def __setup_virtual_machine_done(self, sctx):
        self.log.debug("azure virtual environment %d vm setup done" %
                       sctx.current_job_index)
        ctx = sctx.job_ctxs[sctx.current_job_index]

        # update the status of virtual environment
        ve = self.db.find_first_object_by(VirtualEnvironment,
                                          id=ctx.virtual_environment_id)

        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)
        if ve:
            ve.status = VEStatus.RUNNING

            public_ip, port = adapter.get_virtual_machine_public_endpoint(
                ctx.cloud_service_name, ctx.deployment_name,
                ctx.virtual_machine_name, ctx.endpoint_name)

            if not public_ip:
                self.log.warn(
                    "unable to find public ip for vm %s, set guacamole failed"
                    % ctx.virtual_machine_name)
            else:
                remote_para = get_remote_parameters(ctx.raw_system_config,
                                                    ctx.remote,
                                                    ctx.virtual_machine_name,
                                                    public_ip, port)

                ve.remote_paras = json.dumps(remote_para)

            self.db.commit()
            self.expr_manager.check_expr_status(ve.experiment)

        self.log.debug(
            "azure virtual environment %d vm success callback done, step to next"
            % sctx.current_job_index)
        # step to config next unit
        sctx.current_job_index += 1
        self.__schedule_setup(sctx)
Пример #19
0
    def __setup_virtual_machine_without_deployment_existed(self, sctx):
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        if not ctx.is_vm_image:
            network_config = get_network_config(
                ctx.raw_network_config,
                adapter.get_assigned_endpoints(ctx.cloud_service_name))
        else:
            network_config = None

        try:
            req = adapter.create_virtual_machine_deployment(
                ctx.cloud_service_name,
                ctx.deployment_name,
                ctx.deployment_slot,
                ctx.virtual_machine_label,
                ctx.virtual_machine_name,
                ctx.system_config,
                ctx.os_virtual_hard_disk,
                network_config,
                role_size=ctx.virtual_machine_size,
                vm_image_name=ctx.image_name if ctx.is_vm_image else None)
        except Exception as e:
            self.log.error(
                "azure virtual environment %d create virtual machine %r failed: %r"
                % (sctx.current_job_index, ctx.virtual_machine_name, str(e)))
            self.__on_setup_failed(sctx)
            return

        # wait for add virtual machine to finish
        ctx.request_id = req.request_id
        ctx.vm_need_config = True if ctx.is_vm_image else False
        self.__wait_for_create_virtual_machine_deployment(sctx)
Пример #20
0
    def __setup_virtual_machine_with_deployment_existed(self, sctx):
        # get context from super context
        ctx = sctx.job_ctxs[sctx.current_job_index]

        adapter = VirtualMachineAdapter(ctx.subscription_id,
                                        ctx.pem_url,
                                        host=ctx.management_host)

        deployment_name = adapter.get_deployment_name(ctx.cloud_service_name,
                                                      ctx.deployment_slot)
        if not ctx.is_vm_image:
            network_config = get_network_config(
                ctx.raw_network_config,
                adapter.get_assigned_endpoints(ctx.cloud_service_name))
        else:
            network_config = None

        # add virtual machine to deployment
        try:
            # create if vm is not created
            if not adapter.virtual_machine_exists(ctx.cloud_service_name,
                                                  deployment_name,
                                                  ctx.virtual_machine_name):
                req = adapter.add_virtual_machine(
                    ctx.cloud_service_name,
                    deployment_name,
                    ctx.virtual_machine_name,
                    ctx.system_config,
                    ctx.os_virtual_hard_disk,
                    network_config=network_config,
                    role_size=ctx.virtual_machine_size,
                    vm_image_name=ctx.image_name if ctx.is_vm_image else None)
            # NOTE: if the vm is created, we think is well configured
            # else:  # if vm is created, then we need to config the vm
            #    ctx.vm_need_config = True
            #    self.__wait_for_virtual_machine_ready(sctx)
            #    return
            else:
                self.__setup_virtual_machine_done(sctx)
                return
        except Exception as e:
            self.log.error(
                "azure virtual environment %d create virtual machine %r failed: %r"
                % (sctx.current_job_index, ctx.virtual_machine_name, str(e)))
            self.__on_setup_failed(sctx)
            return

        # wait for add virtual machine to finish
        ctx.request_id = req.request_id
        ctx.vm_need_config = True if ctx.is_vm_image else False
        self.__wait_for_add_virtual_machine(sctx)