Example #1
0
    def __init__(self, agent_config):
        self.logger = logging.getLogger(__name__)

        # If VimClient's housekeeping thread failed to update its own cache,
        # call errback to commit suicide. Watchdog will bring up the agent
        # again.
        errback = lambda: suicide()
        self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
                                    errback=errback)
        atexit.register(lambda client: client.disconnect(), self.vim_client)

        self._uuid = self.vim_client.host_uuid
        self.set_memory_overcommit(agent_config.memory_overcommit)

        image_datastores = [ds["name"] for ds in agent_config.image_datastores]
        self.datastore_manager = EsxDatastoreManager(
            self, agent_config.datastores, agent_config.image_datastores)
        # datastore manager needs to update the cache when there is a change.
        self.vim_client.add_update_listener(self.datastore_manager)
        self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
        self.disk_manager = EsxDiskManager(self.vim_client,
                                           self.datastore_manager)
        self.image_manager = EsxImageManager(self.vim_client,
                                             self.datastore_manager)
        self.network_manager = EsxNetworkManager(self.vim_client,
                                                 agent_config.networks)
        self.system = EsxSystem(self.vim_client)
        self.image_manager.monitor_for_cleanup()
        self.image_transferer = HttpNfcTransferer(self.vim_client,
                                                  image_datastores)
        atexit.register(self.image_manager.cleanup)
    def setUp(self):
        if "host_remote_test" not in config:
            raise SkipTest()

        self.host = config["host_remote_test"]["server"]
        self.pwd = config["host_remote_test"]["esx_pwd"]
        self.agent_port = config["host_remote_test"].get("agent_port", 8835)
        if self.host is None or self.pwd is None:
            raise SkipTest()

        self.image_datastore = config["host_remote_test"].get(
            "image_datastore", "datastore1")

        self._logger = logging.getLogger(__name__)
        self.vim_client = VimClient(self.host, "root", self.pwd)
        self.http_transferer = HttpNfcTransferer(self.vim_client,
                                                 [self.image_datastore],
                                                 self.host)

        with tempfile.NamedTemporaryFile(delete=False) as source_file:
            with open(source_file.name, 'wb') as f:
                f.write(os.urandom(1024 * 100))
        self.random_file = source_file.name

        self.remote_files_to_delete = []
 def setUp(self, connect, creds):
     self.shadow_vm_id = SHADOW_VM_NAME_PREFIX + str(uuid.uuid1())
     self.image_datastores = ["image_ds", "alt_image_ds"]
     creds.return_value = ["username", "password"]
     self.vim_client = VimClient(auto_sync=False)
     self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv")
     self.patcher.start()
     services.register(ServiceName.AGENT_CONFIG, MagicMock())
     self.http_transferer = HttpNfcTransferer(self.vim_client,
                                              self.image_datastores)
 def setUp(self, connect, creds):
     self.host_uuid = str(uuid.uuid4())
     VimClient.host_uuid = self.host_uuid
     self.image_ds = "image_ds"
     creds.return_value = ["username", "password"]
     self.vim_client = VimClient(auto_sync=False)
     self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv")
     self.patcher.start()
     services.register(ServiceName.AGENT_CONFIG, MagicMock())
     self.http_transferer = HttpNfcTransferer(self.vim_client,
                                              self.image_ds)
class TestHttpTransfer(unittest.TestCase):
    """Http Transferer tests."""

    @patch.object(VimClient, "acquire_credentials")
    @patch("pysdk.connect.Connect")
    def setUp(self, connect, creds):
        self.shadow_vm_id = SHADOW_VM_NAME_PREFIX + str(uuid.uuid1())
        self.image_datastores = ["image_ds", "alt_image_ds"]
        creds.return_value = ["username", "password"]
        self.vim_client = VimClient(auto_sync=False)
        self.patcher = patch("host.hypervisor.esx.vm_config.GetEnv")
        self.patcher.start()
        services.register(ServiceName.AGENT_CONFIG, MagicMock())
        self.http_transferer = HttpNfcTransferer(self.vim_client,
                                                 self.image_datastores)

    def tearDown(self):
        self.vim_client.disconnect(wait=True)
        self.patcher.stop()

    @parameterized.expand([
        (True,), (False,)
    ])
    @patch("host.hypervisor.esx.http_disk_transfer.VimClient")
    @patch("host.hypervisor.esx.http_disk_transfer.DirectClient")
    def test_get_remote_connections(self, get_svc_ticket_success, _client_cls,
                                    _vim_client_cls):
        host = "mock_host"
        port = 8835
        get_service_ticket_mock = MagicMock()
        if get_svc_ticket_success:
            get_service_ticket_mock.result = ServiceTicketResultCode.OK
        else:
            get_service_ticket_mock.result = ServiceTicketResultCode.NOT_FOUND
        instance = _client_cls.return_value
        instance.get_service_ticket.return_value = get_service_ticket_mock

        if get_svc_ticket_success:
            (agent_conn,
             vim_conn) = self.http_transferer._get_remote_connections(
                 host, port)
            _client_cls.assert_called_once_with(
                "Host", Host.Client, host, port)
            instance.connect.assert_called_once_with()
            request = ServiceTicketRequest(service_type=ServiceType.VIM)
            instance.get_service_ticket.assert_called_once_with(request)

            _vim_client_cls.assert_called_once_with(
                host=host, ticket=get_service_ticket_mock.vim_ticket,
                auto_sync=False)

            self.assertEqual(agent_conn, instance)
            self.assertEqual(vim_conn, _vim_client_cls.return_value)
        else:
            self.assertRaises(
                ValueError, self.http_transferer._get_remote_connections,
                host, port)

    @parameterized.expand([
        (None, "http://*/ha-nfc/x.vmdk", "http://actual_host/ha-nfc/x.vmdk"),
        (None, "https://*/foo", "https://actual_host/foo"),
        (None, "http://*:1234/foo", "http://*****:*****@patch("uuid.uuid4", return_value="fake_id")
    @patch("pyVmomi.vim.vm.VmImportSpec")
    def test_create_import_vm_spec(self, mock_vm_imp_spec_cls, mock_uuid):
        image_id = "fake_image_id"
        destination_datastore = "fake_datastore"
        create_spec_mock = MagicMock()
        create_empty_disk_mock = MagicMock()

        xferer = self.http_transferer
        xferer._vm_config.create_spec_for_import = create_spec_mock
        xferer._vm_manager.create_empty_disk = create_empty_disk_mock

        spec = xferer._create_import_vm_spec(image_id, destination_datastore)

        expected_vm_id = "fake_id"
        create_spec_mock.assert_called_once_with(
            vm_id=expected_vm_id, image_id=image_id,
            datastore=destination_datastore, memory=32, cpus=1)
        create_empty_disk_mock.assert_called_once_with(
            create_spec_mock.return_value, destination_datastore, None,
            size_mb=1)
        mock_vm_imp_spec_cls.assert_called_once_with(
            configSpec=create_empty_disk_mock.return_value)
        self.assertEqual(spec, mock_vm_imp_spec_cls.return_value)

    def test_get_url_from_import_vm(self):
        host = "mock_host"
        lease_mock = MagicMock()
        url_mock = MagicMock()
        import_spec = MagicMock()
        rp_mock = MagicMock()
        folder_mock = MagicMock()
        vim_client_mock = MagicMock()
        vim_client_mock.host = host

        xferer = self.http_transferer
        xferer._wait_for_lease = MagicMock()
        xferer._get_disk_url_from_lease = MagicMock(return_value=url_mock)
        xferer._ensure_host_in_url = MagicMock(return_value=url_mock)
        vim_client_mock.root_resource_pool = rp_mock
        vim_client_mock.vm_folder = folder_mock
        rp_mock.ImportVApp.return_value = lease_mock

        lease, url = xferer._get_url_from_import_vm(vim_client_mock,
                                                    import_spec)

        rp_mock.ImportVApp.assert_called_once_with(
            import_spec, folder_mock)
        xferer._wait_for_lease.assert_called_once_with(
            lease_mock)
        xferer._get_disk_url_from_lease.assert_called_once_with(lease_mock)
        xferer._ensure_host_in_url.assert_called_once_with(url_mock, host)
        self.assertEqual(lease, lease_mock)
        self.assertEqual(url, url_mock)

    @patch("os.path.exists")
    @patch("__builtin__.open")
    @patch("os.unlink")
    def test_send_image_to_host(self, mock_unlink, mock_open, mock_exists):
        host = "mock_host"
        port = 8835
        image_id = "fake_image_id"
        image_datastore = "fake_image_ds"
        destination_datastore = "fake_destination_image_ds"
        read_lease_mock = MagicMock()
        from_url_mock = MagicMock()
        write_lease_mock = MagicMock()
        to_url_mock = MagicMock()
        import_spec_mock = MagicMock()
        xferer = self.http_transferer

        file_contents = ["fake_metadata", "fake_manifest"]

        def fake_read():
            return file_contents.pop()
        mock_open().__enter__().read.side_effect = fake_read
        mock_exists.return_value = True

        xferer._create_shadow_vm = MagicMock(
            return_value=self.shadow_vm_id)
        xferer._delete_shadow_vm = MagicMock()
        xferer._get_image_stream_from_shadow_vm = MagicMock(
            return_value=(read_lease_mock, from_url_mock))
        xferer.download_file = MagicMock()
        vim_conn_mock = MagicMock()
        agent_conn_mock = MagicMock()
        xferer._get_remote_connections = MagicMock(
            return_value=(agent_conn_mock, vim_conn_mock))
        xferer._create_import_vm_spec = MagicMock(
            return_value=import_spec_mock)
        xferer._get_url_from_import_vm = MagicMock(
            return_value=(write_lease_mock, to_url_mock))
        xferer.upload_file = MagicMock()
        receive_image_resp_mock = MagicMock()
        receive_image_resp_mock.result = ReceiveImageResultCode.OK
        agent_conn_mock.receive_image.return_value = receive_image_resp_mock

        self.http_transferer.send_image_to_host(
            image_id, image_datastore, None, destination_datastore, host, port)

        assert_that(agent_conn_mock.receive_image.call_args_list,
                    has_length(1))
        request = agent_conn_mock.receive_image.call_args_list[0][0][0]
        assert_that(request.metadata, equal_to("fake_metadata"))
        assert_that(request.manifest, equal_to("fake_manifest"))

        xferer._get_image_stream_from_shadow_vm.assert_called_once_with(
            image_id, image_datastore, self.shadow_vm_id)
        expected_tmp_file = "/vmfs/volumes/%s/%s_transfer.vmdk" % (
            self.image_datastores[0], self.shadow_vm_id)
        xferer.download_file.assert_called_once_with(
            from_url_mock, expected_tmp_file, read_lease_mock)
        read_lease_mock.Complete.assert_called_once_with()
        xferer._create_import_vm_spec.assert_called_once_with(
            image_id, destination_datastore)
        xferer._get_url_from_import_vm.assert_called_once_with(
            vim_conn_mock, import_spec_mock)
        xferer.upload_file.assert_called_once_with(
            expected_tmp_file, to_url_mock, write_lease_mock)
        write_lease_mock.Complete.assert_called_once_with()
        xferer._delete_shadow_vm.assert_called_once_with(
            self.shadow_vm_id)
        mock_unlink.assert_called_once_with(expected_tmp_file)
Example #6
0
class EsxHypervisor(object):
    """Manage ESX Hypervisor."""

    def __init__(self, agent_config):
        self.logger = logging.getLogger(__name__)

        # If VimClient's housekeeping thread failed to update its own cache,
        # call errback to commit suicide. Watchdog will bring up the agent
        # again.
        errback = lambda: suicide()
        self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
                                    errback=errback)
        atexit.register(lambda client: client.disconnect(), self.vim_client)

        self._uuid = self.vim_client.host_uuid
        self.set_memory_overcommit(agent_config.memory_overcommit)

        image_datastores = [ds["name"] for ds in agent_config.image_datastores]
        self.datastore_manager = EsxDatastoreManager(
            self, agent_config.datastores, agent_config.image_datastores)
        # datastore manager needs to update the cache when there is a change.
        self.vim_client.add_update_listener(self.datastore_manager)
        self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
        self.disk_manager = EsxDiskManager(self.vim_client,
                                           self.datastore_manager)
        self.image_manager = EsxImageManager(self.vim_client,
                                             self.datastore_manager)
        self.network_manager = EsxNetworkManager(self.vim_client,
                                                 agent_config.networks)
        self.system = EsxSystem(self.vim_client)
        self.image_manager.monitor_for_cleanup()
        self.image_transferer = HttpNfcTransferer(self.vim_client,
                                                  image_datastores)
        atexit.register(self.image_manager.cleanup)

    @property
    def uuid(self):
        return self._uuid

    @property
    def config(self):
        config = gen.hypervisor.esx.ttypes.EsxConfig()
        return TSerialization.serialize(config)

    def normalized_load(self):
        """ Return the maximum of the normalized memory/cpu loads"""
        memory = self.system.memory_info()
        memory_load = memory.used * 100 / memory.total

        # get average cpu load percentage in past 20 seconds
        # since hostd takes a sample in every 20 seconds
        # we use the min 20secs here to get the latest
        # CPU active average over 1 minute
        host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))

        cpu_load = host_stats['rescpu.actav1'] / 100

        return max(memory_load, cpu_load)

    def check_image(self, image_id, datastore_id):
        return self.image_manager.check_image(
            image_id, self.datastore_manager.datastore_name(datastore_id)
        )

    def acquire_vim_ticket(self):
        return self.vim_client.acquire_clone_ticket()

    def acquire_cgi_ticket(self, url, op):
        return self.vim_client.acquire_cgi_ticket(url, op)

    def add_update_listener(self, listener):
        self.vim_client.add_update_listener(listener)

    def remove_update_listener(self, listener):
        self.vim_client.remove_update_listener(listener)

    def transfer_image(self, source_image_id, source_datastore,
                       destination_image_id, destination_datastore,
                       host, port):
        return self.image_transferer.send_image_to_host(
            source_image_id, source_datastore,
            destination_image_id, destination_datastore, host, port)

    def receive_image(self, image_id, datastore, imported_vm_name):
        self.image_manager.receive_image(
            image_id, datastore, imported_vm_name)

    def set_memory_overcommit(self, memory_overcommit):
        # Enable/Disable large page support. If this host is removed
        # from the deployment, large page support will need to be
        # explicitly updated by the user.
        disable_large_pages = memory_overcommit > 1.0
        self.vim_client.set_large_page_support(disable=disable_large_pages)
class TestHttpTransfer(unittest.TestCase):
    def setUp(self):
        if "host_remote_test" not in config:
            raise SkipTest()

        self.host = config["host_remote_test"]["server"]
        self.pwd = config["host_remote_test"]["esx_pwd"]
        self.agent_port = config["host_remote_test"].get("agent_port", 8835)
        if self.host is None or self.pwd is None:
            raise SkipTest()

        self.image_datastore = config["host_remote_test"].get(
            "image_datastore", "datastore1")

        self._logger = logging.getLogger(__name__)
        self.vim_client = VimClient(self.host, "root", self.pwd)
        self.http_transferer = HttpNfcTransferer(self.vim_client,
                                                 [self.image_datastore],
                                                 self.host)

        with tempfile.NamedTemporaryFile(delete=False) as source_file:
            with open(source_file.name, 'wb') as f:
                f.write(os.urandom(1024 * 100))
        self.random_file = source_file.name

        self.remote_files_to_delete = []

    def _cleanup_remote_files(self):
        file_manager = self.vim_client._content.fileManager
        for ds_path in self.remote_files_to_delete:
            try:
                delete_task = file_manager.DeleteFile(ds_path, None)
                task.WaitForTask(delete_task)
            except:
                pass

    def tearDown(self):
        os.unlink(self.random_file)
        self._cleanup_remote_files()
        self.vim_client.disconnect(wait=True)

    def _remote_ds_path(self, ds, relpath):
        return '[%s] %s' % (ds, relpath)

    def _datastore_path_url(self, datastore, relpath):
        quoted_dc_name = 'ha%252ddatacenter'
        url = 'https://%s/folder/%s?dcPath=%s&dsName=%s' % (
            self.host, relpath, quoted_dc_name, datastore)
        return url

    def test_download_missing_file(self):
        url = self._datastore_path_url(self.image_datastore,
                                       "_missing_file_.bin")
        ticket = self.http_transferer._get_cgi_ticket(
            self.host, self.agent_port, url, http_op=HttpOp.GET)
        with tempfile.NamedTemporaryFile(delete=True) as local_file:
            self.assertRaises(TransferException,
                              self.http_transferer.download_file, url,
                              local_file.name, MagicMock(), ticket=ticket)

    def test_upload_file_bad_destination(self):
        url = self._datastore_path_url("_missing__datastore_",
                                       "random.bin")
        ticket = self.http_transferer._get_cgi_ticket(
            self.host, self.agent_port, url, http_op=HttpOp.PUT)
        self.assertRaises(
            TransferException, self.http_transferer.upload_file,
            self.random_file, url, MagicMock(), ticket=ticket)

    def test_raw_file_transfer_roundtrip(self):
        relpath = "_test_http_xfer_random.bin"
        url = self._datastore_path_url(self.image_datastore, relpath)
        ticket = self.http_transferer._get_cgi_ticket(
            self.host, self.agent_port, url, http_op=HttpOp.PUT)
        self.http_transferer.upload_file(self.random_file, url, MagicMock(), ticket=ticket)

        self.remote_files_to_delete.append(
            self._remote_ds_path(self.image_datastore, relpath))

        ticket = self.http_transferer._get_cgi_ticket(
            self.host, self.agent_port, url, http_op=HttpOp.GET)
        with tempfile.NamedTemporaryFile(delete=True) as downloaded_file:
            self.http_transferer.download_file(url, downloaded_file.name,
                                               MagicMock(), ticket=ticket)
            # check that file uploaded and immediately downloaded back is
            # identical to the source file used.
            assert_that(
                filecmp.cmp(self.random_file, downloaded_file.name,
                            shallow=False),
                is_(True))

    @patch('os.path.exists', return_value=True)
    def test_get_streamoptimized_image_stream(self, _exists):
        image_id = "ttylinux"
        shadow_vm_id = self.http_transferer._create_shadow_vm()
        lease, url = self.http_transferer._get_image_stream_from_shadow_vm(
            image_id, self.image_datastore, shadow_vm_id)
        try:
            with tempfile.NamedTemporaryFile(delete=True) as downloaded_file:
                # see if we can download without errors
                self.http_transferer.download_file(url, downloaded_file.name, lease)
                # check that the first part of the file looks like that from a
                # stream-optimized disk
                with open(downloaded_file.name, 'rb') as f:
                    data = f.read(65536)
                    assert_that(len(data), is_(65536))
                    regex = re.compile("streamOptimized",
                                       re.IGNORECASE | re.MULTILINE)
                    matches = regex.findall(data)
                    assert_that(matches, not(empty()))
        finally:
            lease.Complete()
Example #8
0
class EsxHypervisor(object):
    """Manage ESX Hypervisor."""

    def __init__(self, agent_config):
        self.logger = logging.getLogger(__name__)

        # If VimClient's housekeeping thread failed to update its own cache,
        # call errback to commit suicide. Watchdog will bring up the agent
        # again.
        self.vim_client = VimClient(wait_timeout=agent_config.wait_timeout,
                                    errback=lambda: suicide())
        atexit.register(lambda client: client.disconnect(), self.vim_client)

        self._uuid = self.vim_client.host_uuid

        self.datastore_manager = EsxDatastoreManager(
            self, agent_config.datastores, agent_config.image_datastores)
        # datastore manager needs to update the cache when there is a change.
        self.vim_client.add_update_listener(self.datastore_manager)
        self.vm_manager = EsxVmManager(self.vim_client, self.datastore_manager)
        self.disk_manager = EsxDiskManager(self.vim_client,
                                           self.datastore_manager)
        self.image_manager = EsxImageManager(self.vim_client,
                                             self.datastore_manager)
        self.network_manager = EsxNetworkManager(self.vim_client,
                                                 agent_config.networks)
        self.system = EsxSystem(self.vim_client)
        self.image_manager.monitor_for_cleanup()
        self.image_transferer = HttpNfcTransferer(
                self.vim_client,
                self.datastore_manager.image_datastores())
        atexit.register(self.image_manager.cleanup)

    @property
    def uuid(self):
        return self._uuid

    def check_image(self, image_id, datastore_id):
        return self.image_manager.check_image(
            image_id, self.datastore_manager.datastore_name(datastore_id)
        )

    def acquire_vim_ticket(self):
        return self.vim_client.acquire_clone_ticket()

    def add_update_listener(self, listener):
        self.vim_client.add_update_listener(listener)

    def remove_update_listener(self, listener):
        self.vim_client.remove_update_listener(listener)

    def transfer_image(self, source_image_id, source_datastore,
                       destination_image_id, destination_datastore,
                       host, port):
        return self.image_transferer.send_image_to_host(
            source_image_id, source_datastore,
            destination_image_id, destination_datastore, host, port)

    def prepare_receive_image(self, image_id, datastore):
        return self.image_manager.prepare_receive_image(image_id, datastore)

    def receive_image(self, image_id, datastore, imported_vm_name, metadata):
        self.image_manager.receive_image(image_id, datastore, imported_vm_name, metadata)

    def set_memory_overcommit(self, memory_overcommit):
        # Enable/Disable large page support. If this host is removed
        # from the deployment, large page support will need to be
        # explicitly updated by the user.
        disable_large_pages = memory_overcommit > 1.0
        self.vim_client.set_large_page_support(disable=disable_large_pages)