def test_hostd_rescpu_actav1_match_vsi_value(self):
        try:
            vsi = VsiWrapper()
        except:
            return

        cpu_load_history = vsi.get(
            "/sched/groups/0/stats"
            "/cpuStatsDir/cpuLoadHistory"
            "/cpuLoadHistory1MinInPct")
        vsi_cpu_load = cpu_load_history["avgActive"]

        # get average cpu load percentage in past 20 seconds
        # since hostd takes a sample in every 20 seconds
        # we use the min 20secs here to get the latest
        # CPU active average over 1 minute
        host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
        rescpu_cpu_load = host_stats['rescpu.actav1'] / 100
        check_value = False

        # vsi gets the current cpu active average over 1 minute.
        # hostd gets the cpu active average over 1 minute 20 seconds ago.
        # Thus if there's a CPU active average boost during the
        # past 20 seconds, the value from hostd's CPU active average
        # value will be 20 seconds late which will have a large deviation.
        if (1 if vsi_cpu_load - 7 < 1 else vsi_cpu_load - 7) \
                <= rescpu_cpu_load \
                <= vsi_cpu_load + 7:
            check_value = True

        self.assertEqual(check_value, True)
Esempio n. 2
0
    def test_hostd_rescpu_actav1_match_vsi_value(self):
        try:
            vsi = VsiWrapper()
        except:
            return

        cpu_load_history = vsi.get("/sched/groups/0/stats"
                                   "/cpuStatsDir/cpuLoadHistory"
                                   "/cpuLoadHistory1MinInPct")
        vsi_cpu_load = cpu_load_history["avgActive"]

        # get average cpu load percentage in past 20 seconds
        # since hostd takes a sample in every 20 seconds
        # we use the min 20secs here to get the latest
        # CPU active average over 1 minute
        host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
        rescpu_cpu_load = host_stats['rescpu.actav1'] / 100
        check_value = False

        # vsi gets the current cpu active average over 1 minute.
        # hostd gets the cpu active average over 1 minute 20 seconds ago.
        # Thus if there's a CPU active average boost during the
        # past 20 seconds, the value from hostd's CPU active average
        # value will be 20 seconds late which will have a large deviation.
        if (1 if vsi_cpu_load - 7 < 1 else vsi_cpu_load - 7) \
                <= rescpu_cpu_load \
                <= vsi_cpu_load + 7:
            check_value = True

        self.assertEqual(check_value, True)
Esempio n. 3
0
    def TestNamespaceDetails(self, nsdetails):
        """
      Test NVDIMM Namespace information
      """
        global vsi

        self.assertIsNotNone(nsdetails,
                             "NVDIMM Namespace details should not be None")

        # Loop lists to check if namespace details are the same
        for d in nsdetails:
            # Get NVDIMM interleave set properties through VSI call
            v = vsi.get('/hardware/nvd/namespace/%s/details' % (d.uuid))
            self.assertIsNotNone(
                v,
                "NVDIMM namespace details obtained via VSI should not be None")

            discover = {'friendlyName': d.friendlyName, 'guid': d.uuid}
            self.assertEqual(
                discover, v['discovery'],
                "Namespace discovery did not match vsi"
                " namespace details")

            self.assertEqual(
                d.size, v['size'], "Namespace size did not match vsi"
                " namespace details")
            self.checkNamespaceType(v['type'], d.type)
            self.checkNsHealthStatus(v['healthStatus'],
                                     d.namespaceHealthStatus)
            self.assertEqual(
                d.interleavesetID, v['interleavesetID'],
                "Namespace interleaveset ID did not match vsi"
                " namespace details")
            self.checkNamespaceState(v['state'], d.state)
Esempio n. 4
0
    def TestSummary(self, summary):
        """
      Gets summary of NVDIMMs in the system
      """
        global vsi

        self.assertIsNotNone(summary, "NVD summary should not be None")

        # Get NVDIMM Summary through VSI call
        summaryVsi = vsi.get("/hardware/nvd/system/summary")
        self.assertIsNotNone(summaryVsi,
                             "Summary obtained via VSI should not be None")

        self.assertEqual(summaryVsi['numDimms'], summary.numDimms,
                         "Number of dimms did not match vsi summary")

        self.checkDeviceHealth(summaryVsi['healthStatus'],
                               summary.healthStatus)

        self.assertEqual(summaryVsi['totalCapacity'], summary.totalCapacity,
                         "Total capacity of dimms did not match vsi summary")
        self.assertEqual(
            summaryVsi['persistentCapacity'], summary.persistentCapacity,
            "Persistent capacity of dimms did not match vsi summary")
        self.assertEqual(
            summaryVsi['availablePersistentCapacity'],
            summary.availableCapacity,
            "Available persistent capacity of dimms did not match"
            " vsi summary")
        self.assertEqual(
            summaryVsi['numInterleavesets'], summary.numInterleavesets,
            "Number of interleave sets did not match vsi summary")
        self.assertEqual(summaryVsi['numNamespaces'], summary.numNamespaces,
                         "Number of namespaces did not match vsi summary")
Esempio n. 5
0
def buildList():
    """Build list of ssds

   """
    ssds = {}
    node = "/vmkModules/plog/devices/"
    for disk in vsi.list(node):
        isSSD = vsi.get(node + disk + "/info")["isSSD"]
        if isSSD:
            elevNode = node + disk + "/elevStats"
            ssds[disk] = {
                "state": "QUERY",
                "stateval": 0,
                "prevval": vsi.get(elevNode)["plogDataUsage"],
                "node": elevNode,
            }
    return ssds
Esempio n. 6
0
def checkDrainState(ssds):
    """Update list of ssds

   """
    done = True
    for (diskName, diskInfo) in list(ssds.items()):
        _logger.info(diskInfo)
        if diskInfo["state"] == "QUERY":
            done = False
            fullData = vsi.get(diskInfo["node"])
            newVal = fullData["plogDataUsage"]
            prevVal = diskInfo["prevval"]
            _logger.info("Prev Value: %d, New Value: %d", newVal, prevVal)
            rate = prevVal - newVal
            if rate >= 0:
                rate /= _INTERVAL
                _logger.info(
                    "Disk %s plog data drain rate %d MB",
                    diskName,
                    rate / (1024 * 1024),
                )
            elif newVal >= 10000000000000000000:
                pass
            else:
                _logger.error(
                    "Disk %s plog data increasing %d MB to %d MB",
                    diskName,
                    prevVal >> 20,
                    newVal >> 20,
                )
                if rate <= -10 * 1024 * 1024:  # Too much new data. raise
                    raise Exception("Disk %s plog data increasing" % diskName)
            _logger.debug(
                "Disk %s waiting. Usage %d MB, DrainRate %d MB",
                diskName,
                newVal / (1024 * 1024),
                rate / (1024 * 1024),
            )
            diskInfo["prevval"] = newVal
            if newVal <= _DATA_THRESH or newVal >= 10000000000000000000:  # or rate <= _RATE_THRESH:
                diskInfo["state"] = "WAIT"
                diskInfo["stateval"] = 30  # 30 seconds in wait state
                _logger.debug("Disk %s drain end condition met", diskName)
            continue

        if diskInfo["state"] == "WAIT":
            diskInfo["stateval"] -= _INTERVAL
            if diskInfo["stateval"] <= 0:
                diskInfo["state"] = "DONE"
            else:
                done = False

    return done
Esempio n. 7
0
    def TestInterleavesetInfo(self, setinfo):
        """
      Test NVDIMM Interleaveset information
      """
        global vsi

        self.assertIsNotNone(
            setinfo, "NVDIMM Interleave set properties should not be None")

        # Loop lists to check if interleave set property contents are the same
        for s in setinfo:
            # Get NVDIMM interleave set properties through VSI call
            setidstr = "{0:#0{1}x}".format(s.setId, 10)
            v = vsi.get('/hardware/nvd/interleaveset/%s/properties' % setidstr)
            self.assertIsNotNone(
                v, "NVDIMM interleave set properties obtained via VSI should"
                " not be None")

            self.assertEqual(
                s.setId, v['id'],
                "Interleave set ID did not match vsi set property")
            self.checkRangeType(v['type'], s.rangeType)
            self.assertEqual(
                s.baseAddress, v['baseAddress'],
                "Interleave set Base Address did not match vsi"
                " set property")
            self.assertEqual(
                s.size, v['size'],
                "Interleave set size did not match vsi set property")
            self.assertEqual(
                s.availableSize, v['availableSize'],
                "Interleave set available size did not match vsi"
                " set property")

            self.checkInterleaveSetState(v['state'], s.state)
            dimmList = vsi.list('/hardware/nvd/interleaveset/%s/dimm' %
                                setidstr)
            self.assertIsNotNone(
                dimmList,
                "List of dimms part of interleave set obtained via VSI should"
                " not be None")

            for d, l in zip(s.deviceList, dimmList):
                # Match device list
                dimmstr = "{0:#0{1}x}".format(d, 10)
                self.assertEqual(
                    dimmstr, l, "Interleave set dimm ID did not match vsi"
                    " device list")
Esempio n. 8
0
def handleVmciRequests(port):
    VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
    ECONNABORTED = 103 # Error on non privileged client

    bsize = MAX_JSON_SIZE
    txt = create_string_buffer(bsize)

    cartel = c_int32()
    sock = lib.vmci_init(c_uint(port))
    if sock == VMCI_ERROR:
        errno = get_errno()
        raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
                        %  (os.strerror(errno), errno))

    skip_count = MAX_SKIP_COUNT  # retries for vmci_get_one_op failures
    while True:
        c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
        logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
                      c, txt.value)

        errno = get_errno()
        if errno == ECONNABORTED:
            logging.warn("Client with non privileged port attempted a request")
            continue
        if c == VMCI_ERROR:
            # We can self-correct by reoping sockets internally. Give it a chance.
            logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
                            c, os.strerror(errno), errno)
            skip_count = skip_count - 1
            if skip_count <= 0:
                raise Exception(
                    "vmci_get_one_op: too many errors. Giving up.")
            continue
        else:
            skip_count = MAX_SKIP_COUNT  # reset the counter, just in case

        # Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
        vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
                            str(cartel.value))
        group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)

        vm_name = group_info["displayName"]
        cfg_path = group_info["cfgPath"]
        uuid = group_info["uuid"]
        # pyVmomi expects uuid like this one: 564dac12-b1a0-f735-0df3-bceb00b30340
        # to get it from uuid in VSI vms/<id>/vmmGroup, we use the following format:
        UUID_FORMAT = "{0}{1}{2}{3}-{4}{5}-{6}{7}-{8}{9}-{10}{11}{12}{13}{14}{15}"
        vm_uuid = UUID_FORMAT.format(*uuid.replace("-",  " ").split())

        try:
            req = json.loads(txt.value.decode('utf-8'))
        except ValueError as e:
            ret = {u'Error': "Failed to parse json '%s'." % txt.value}
        else:
            details = req["details"]
            opts = details["Opts"] if "Opts" in details else {}
            threading.currentThread().setName(vm_name)
            ret = executeRequest(vm_uuid=vm_uuid,
                                 vm_name=vm_name,
                                 config_path=cfg_path,
                                 cmd=req["cmd"],
                                 full_vol_name=details["Name"],
                                 opts=opts)
            logging.info("executeRequest '%s' completed with ret=%s",
                         req["cmd"], ret)

        ret_string = json.dumps(ret)
        response = lib.vmci_reply(c, c_char_p(ret_string.encode()))
        errno = get_errno()
        logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
        if response == VMCI_ERROR:
            logging.warning("vmci_reply returned error %s (errno=%d)",
                            os.strerror(errno), errno)

    lib.close(sock)  # close listening socket when the loop is over
Esempio n. 9
0
def dropWrite(wait=None):
    """Drop write cache

   wait: wait for dropCache to complete
              None => return immediately, DO NOT reset nodes
              0    => wait till drop is complete. RESET nodes
              other =>

   """
    _logger.debug("Dropping write cache")
    oldThresh = vsi.get("/vmkModules/plog/plogLogThreshold")
    defRunElev = vsi.get("/config/LSOM/intOpts/plogRunElevator")["cur"]
    oldSleep = vsi.get("/config/LSOM/intOpts/plogElevSleepTime")["cur"]
    if oldThresh < 1000:
        _logger.error("Skipping write cache as log thresh is very small %d",
                      oldThresh)
        return

    _logger.warning(
        "Reset using the following commands after drop cache completes")
    _logger.warning("vsish -e set /config/LSOM/intOpts/plogRunElevator %d",
                    defRunElev)
    _logger.warning("vsish -e set /config/LSOM/intOpts/plogElevSleepTime %d",
                    oldSleep)
    defDdpThrottle = None
    try:
        defDdpThrottle = vsi.get(
            "/config/LSOM/intOpts/dedupElevThrottle")["cur"]
        _logger.warning(
            "vsish -e set /config/LSOM/intOpts/dedupElevThrottle %d",
            defDdpThrottle,
        )
        vsi.set("/config/LSOM/intOpts/dedupElevThrottle", 0)
    except:
        _logger.warning(
            "vsi /config/LSOM/intOpts/dedupElevThrottle does not exist for this build"
        )
        pass

    vsi.set("/config/LSOM/intOpts/plogRunElevator", 1)
    vsi.set("/config/LSOM/intOpts/plogElevSleepTime", 1)

    if wait is None:
        _logger.warning("Returning without resetting vsi nodes")
        return

    start = time.time()
    end = 0
    if wait > 0:
        end = start + wait
    try:
        ssds = buildList()
        while True:
            attempts = 0
            while attempts < 10:
                try:
                    done = checkDrainState(ssds)
                    break
                except:
                    attempts = attempts + 1
                    _logger.warning("dropcache increase for %r times" %
                                    attempts)
                    time.sleep(60)
            if done:
                _logger.warning("Drop cache completed")
                break
            if end and time.time() > end:
                _logger.warning("Drop cache timed out (%d s). Resetting nodes",
                                wait)
                break
            time.sleep(_INTERVAL)
    finally:
        _logger.warning(
            "dropWrite returning in %d s, Resetting vsi nodes",
            time.time() - start,
        )
        vsi.set("/config/LSOM/intOpts/plogRunElevator", defRunElev)
        vsi.set("/config/LSOM/intOpts/plogElevSleepTime", oldSleep)
        try:
            vsi.set("/config/LSOM/intOpts/dedupElevThrottle", defDdpThrottle)
        except:
            _logger.warning(
                "vsi /config/LSOM/intOpts/dedupElevThrottle does not exist for this build"
            )
            pass
Esempio n. 10
0
def handleVmciRequests(port):
    VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
    ECONNABORTED = 103 # Error on non privileged client

    bsize = MAX_JSON_SIZE
    txt = create_string_buffer(bsize)

    cartel = c_int32()
    sock = lib.vmci_init(c_uint(port))
    if sock == VMCI_ERROR:
        errno = get_errno()
        raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
                        %  (os.strerror(errno), errno))

    skip_count = MAX_SKIP_COUNT  # retries for vmci_get_one_op failures
    while True:
        c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
        logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
                      c, txt.value)

        errno = get_errno()
        if errno == ECONNABORTED:
            logging.warn("Client with non privileged port attempted a request")
            continue
        if c == VMCI_ERROR:
            # We can self-correct by reoping sockets internally. Give it a chance.
            logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
                            c, os.strerror(errno), errno)
            skip_count = skip_count - 1
            if skip_count <= 0:
                raise Exception(
                    "vmci_get_one_op: too many errors. Giving up.")
            continue
        else:
            skip_count = MAX_SKIP_COUNT  # reset the counter, just in case

        # Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
        vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
                            str(cartel.value))
        group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)

        vm_name = group_info["displayName"]
        cfg_path = group_info["cfgPath"]
        uuid = group_info["uuid"]
        # pyVmomi expects uuid like this one: 564dac12-b1a0-f735-0df3-bceb00b30340
        # to get it from uuid in VSI vms/<id>/vmmGroup, we use the following format:
        UUID_FORMAT = "{0}{1}{2}{3}-{4}{5}-{6}{7}-{8}{9}-{10}{11}{12}{13}{14}{15}"
        vm_uuid = UUID_FORMAT.format(*uuid.replace("-",  " ").split())

        try:
            req = json.loads(txt.value.decode('utf-8'))
        except ValueError as e:
            ret = {u'Error': "Failed to parse json '%s'." % txt.value}
        else:
            details = req["details"]
            opts = details["Opts"] if "Opts" in details else {}
            threading.currentThread().setName(vm_name)
            ret = executeRequest(vm_uuid=vm_uuid,
                                 vm_name=vm_name,
                                 config_path=cfg_path,
                                 cmd=req["cmd"],
                                 full_vol_name=details["Name"],
                                 opts=opts)
            logging.info("executeRequest '%s' completed with ret=%s",
                         req["cmd"], ret)

        ret_string = json.dumps(ret)
        response = lib.vmci_reply(c, c_char_p(ret_string.encode()))
        errno = get_errno()
        logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
        if response == VMCI_ERROR:
            logging.warning("vmci_reply returned error %s (errno=%d)",
                            os.strerror(errno), errno)

    lib.close(sock)  # close listening socket when the loop is over
Esempio n. 11
0

import vmware.vsi as vsi
import time
from collections import defaultdict

def calculate(xyz,nics):
        if xyz[0] == 0 and xyz[1] == 0:
                print "Lets wait for packets to come to %s" %(nics)
        else:
                print "The number of packets received for %s in the last 10 seconds are " %(nics.upper())+ str(xyz[2]-xyz[0])
                print "The number of packets sent from %s in the last 10 seconds are " %(nics.upper())+ str(xyz[3]-xyz[1])


vmnics = defaultdict(list)
no_nics=vsi.list('/net/pNics/')
print "There are total of %d network cards in the machine " %(len(no_nics))
for i in no_nics:
        vmnics[i]=[0,0]
while True:
        for nics in no_nics:
                ReceivePackets=vsi.get('/net/pNics/%s/stats' % nics)['rxpkt']
                vmnics[nics].append(ReceivePackets)
                SendPackets=vsi.get('/net/pNics/%s/stats' % nics)['txpkt']
                vmnics[nics].append(SendPackets)
                calculate(vmnics[nics],nics)
                del vmnics[nics][0:2]
	print "-"*20 +  "Sleeping for 10 seconds" + "-"*20
	print 2*'\n'
        time.sleep(10)
Esempio n. 12
0
import time
from collections import defaultdict


def calculate(xyz, nics):
    if xyz[0] == 0 and xyz[1] == 0:
        print "Lets wait for packets to come to %s" % (nics)
    else:
        print "The number of packets received for %s in the last 10 seconds are " % (
            nics.upper()) + str(xyz[2] - xyz[0])
        print "The number of packets sent from %s in the last 10 seconds are " % (
            nics.upper()) + str(xyz[3] - xyz[1])


vmnics = defaultdict(list)
no_nics = vsi.list('/net/pNics/')
print "There are total of %d network cards in the machine " % (len(no_nics))
for i in no_nics:
    vmnics[i] = [0, 0]
while True:
    for nics in no_nics:
        ReceivePackets = vsi.get('/net/pNics/%s/stats' % nics)['rxpkt']
        vmnics[nics].append(ReceivePackets)
        SendPackets = vsi.get('/net/pNics/%s/stats' % nics)['txpkt']
        vmnics[nics].append(SendPackets)
        calculate(vmnics[nics], nics)
        del vmnics[nics][0:2]
    print "-" * 20 + "Sleeping for 10 seconds" + "-" * 20
    print 2 * '\n'
    time.sleep(10)
Esempio n. 13
0
    def TestDimmInfo(self, dimminfo):
        """
      Test NVDIMM Information
      """
        global vsi

        self.assertIsNotNone(dimminfo, "NVDIMM Information should not be None")

        # Loop lists to check if dimminfo contents are the same
        for d in dimminfo:
            # Get NVDIMM Information through VSI call
            dimmstr = "{0:#0{1}x}".format(d.dimmHandle, 10)
            v = vsi.get('/hardware/nvd/dimm/%s/dimminfo' % dimmstr)
            self.assertIsNotNone(
                v, "NVDIMM information obtained via VSI should"
                " not be None")

            self.assertEqual(d.dimmHandle, v['dimmHandle'],
                             "Dimm Handle did not match vsi dimm handle")

            self.assertEqual(
                d.totalCapacity, v['totalCapacity'],
                "Total Capacity did not match vsi dimm information")
            self.assertEqual(
                d.persistentCapacity, v['persistentCapacity'],
                "Persistent Capacity did not match vsi"
                " dimm information")
            self.assertEqual(
                d.availablePersistentCapacity,
                v['availablePersistentCapacity'],
                "Available Persistent Capacity did not match vsi"
                " dimm information")
            self.assertEqual(
                d.volatileCapacity, v['volatileCapacity'],
                "Volatile Capacity did not match vsi dimm"
                " information")
            self.assertEqual(
                d.availableVolatileCapacity, v['availableVolatileCapacity'],
                "Available Volatile Capacity did not match vsi"
                " dimm information")

            # Get list of regions part of DIMM
            regionList = vsi.list('/hardware/nvd/dimm/%s/region' % dimmstr)
            self.assertIsNotNone(regionList, "Region list cannot be none")

            stateFlags = 0
            """
         Get vsi region information and match with hostd obtained
         region information.
         """
            for vsiregion, r in zip(regionList, d.regionInfo):
                regionInfo = vsi.get(
                    '/hardware/nvd/dimm/%s/region/%s/regioninfo' %
                    (dimmstr, vsiregion))
                self.assertIsNotNone(regionInfo,
                                     "Region information cannot be none")

                self.assertEqual(r.regionId, regionInfo['id'],
                                 "Region ID does not match")
                self.assertEqual(r.setId, regionInfo['setId'],
                                 "Setid does not match vsi information")
                self.checkRangeType(regionInfo['type'], r.rangeType)
                self.assertEqual(
                    r.startAddr, regionInfo['startAddr'],
                    "Region start address does not match vsi information")
                self.assertEqual(r.size, regionInfo['size'],
                                 "Region Size does not match vsi information")
                self.assertEqual(
                    r.offset, regionInfo['offset'],
                    "Region offset does not match vsi information")
                stateFlags = stateFlags | regionInfo['stateFlags']

            ## Get NVDIMM topology
            vsiTopo = vsi.get('/hardware/nvd/dimm/%s/topology' % dimmstr)
            ## Test NVDIMM representation string
            self.assertIsNotNone(vsiTopo, "Topology can not be null")
            self.assertEqual(
                vsiTopo['representationStr'], d.representationString,
                "Representation string does not match vsi information")

            ## Check health information for DIMM
            vsiHealth = vsi.get('/hardware/nvd/dimm/%s/healthinfo' % dimmstr)
            self.assertIsNotNone(vsiHealth,
                                 "Health information cannot be none")
            self.TestDimmHealthinfo(stateFlags, d.healthInfo, vsiHealth)
Esempio n. 14
0
def handleVmciRequests():
    VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
    # Load and use DLL with vsocket shim to listen for docker requests
    lib = CDLL(os.path.join(LIB_LOC, "libvmci_srv.so"), use_errno=True)

    bsize = MAX_JSON_SIZE
    txt = create_string_buffer(bsize)

    cartel = c_int32()
    sock = lib.vmci_init()
    if sock == VMCI_ERROR:
        errno = get_errno()
        raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
                        %  (os.strerror(errno), errno))

    skip_count = MAX_SKIP_COUNT  # retries for vmci_get_one_op failures
    while True:
        c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
        logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
                      c, txt.value)

        if c == VMCI_ERROR:
            # We can self-correct by reoping sockets internally. Give it a chance.
            errno = get_errno()
            logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
                            c, os.strerror(errno), errno)
            skip_count = skip_count - 1
            if skip_count <= 0:
                raise Exception(
                    "vmci_get_one_op: too many errors. Giving up.")
            continue
        else:
            skip_count = MAX_SKIP_COUNT  # reset the counter, just in case

        # Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
        vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
                            str(cartel.value))
        group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)

        vm_name = group_info["displayName"]
        cfg_path = group_info["cfgPath"]

        try:
            req = json.loads(txt.value, "utf-8")
        except ValueError as e:
            ret = {u'Error': "Failed to parse json '%s'." % txt.value}
        else:
            details = req["details"]
            opts = details["Opts"] if "Opts" in details else {}
            ret = executeRequest(vm_name, cfg_path, req["cmd"],
                                 details["Name"], opts)
            logging.info("executeRequest '%s' completed with ret=%s",
                         req["cmd"], ret)

        response = lib.vmci_reply(c, c_char_p(json.dumps(ret)))
        errno = get_errno()
        logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
        if response == VMCI_ERROR:
            logging.warning("vmci_reply returned error %s (errno=%d)",
                            os.strerror(errno), errno)

    lib.close(sock)  # close listening socket when the loop is over