def test_hostd_rescpu_actav1_match_vsi_value(self):
try:
vsi = VsiWrapper()
except:
return
cpu_load_history = vsi.get(
"/sched/groups/0/stats"
"/cpuStatsDir/cpuLoadHistory"
"/cpuLoadHistory1MinInPct")
vsi_cpu_load = cpu_load_history["avgActive"]
# get average cpu load percentage in past 20 seconds
# since hostd takes a sample in every 20 seconds
# we use the min 20secs here to get the latest
# CPU active average over 1 minute
host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
rescpu_cpu_load = host_stats['rescpu.actav1'] / 100
check_value = False
# vsi gets the current cpu active average over 1 minute.
# hostd gets the cpu active average over 1 minute 20 seconds ago.
# Thus if there's a CPU active average boost during the
# past 20 seconds, the value from hostd's CPU active average
# value will be 20 seconds late which will have a large deviation.
if (1 if vsi_cpu_load - 7 < 1 else vsi_cpu_load - 7) \
<= rescpu_cpu_load \
<= vsi_cpu_load + 7:
check_value = True
self.assertEqual(check_value, True)
def test_hostd_rescpu_actav1_match_vsi_value(self):
try:
vsi = VsiWrapper()
except:
return
cpu_load_history = vsi.get("/sched/groups/0/stats"
"/cpuStatsDir/cpuLoadHistory"
"/cpuLoadHistory1MinInPct")
vsi_cpu_load = cpu_load_history["avgActive"]
# get average cpu load percentage in past 20 seconds
# since hostd takes a sample in every 20 seconds
# we use the min 20secs here to get the latest
# CPU active average over 1 minute
host_stats = copy.copy(self.vim_client.get_perf_manager_stats(20))
rescpu_cpu_load = host_stats['rescpu.actav1'] / 100
check_value = False
# vsi gets the current cpu active average over 1 minute.
# hostd gets the cpu active average over 1 minute 20 seconds ago.
# Thus if there's a CPU active average boost during the
# past 20 seconds, the value from hostd's CPU active average
# value will be 20 seconds late which will have a large deviation.
if (1 if vsi_cpu_load - 7 < 1 else vsi_cpu_load - 7) \
<= rescpu_cpu_load \
<= vsi_cpu_load + 7:
check_value = True
self.assertEqual(check_value, True)
def TestNamespaceDetails(self, nsdetails):
"""
Test NVDIMM Namespace information
"""
global vsi
self.assertIsNotNone(nsdetails,
"NVDIMM Namespace details should not be None")
# Loop lists to check if namespace details are the same
for d in nsdetails:
# Get NVDIMM interleave set properties through VSI call
v = vsi.get('/hardware/nvd/namespace/%s/details' % (d.uuid))
self.assertIsNotNone(
v,
"NVDIMM namespace details obtained via VSI should not be None")
discover = {'friendlyName': d.friendlyName, 'guid': d.uuid}
self.assertEqual(
discover, v['discovery'],
"Namespace discovery did not match vsi"
" namespace details")
self.assertEqual(
d.size, v['size'], "Namespace size did not match vsi"
" namespace details")
self.checkNamespaceType(v['type'], d.type)
self.checkNsHealthStatus(v['healthStatus'],
d.namespaceHealthStatus)
self.assertEqual(
d.interleavesetID, v['interleavesetID'],
"Namespace interleaveset ID did not match vsi"
" namespace details")
self.checkNamespaceState(v['state'], d.state)
def TestSummary(self, summary):
"""
Gets summary of NVDIMMs in the system
"""
global vsi
self.assertIsNotNone(summary, "NVD summary should not be None")
# Get NVDIMM Summary through VSI call
summaryVsi = vsi.get("/hardware/nvd/system/summary")
self.assertIsNotNone(summaryVsi,
"Summary obtained via VSI should not be None")
self.assertEqual(summaryVsi['numDimms'], summary.numDimms,
"Number of dimms did not match vsi summary")
self.checkDeviceHealth(summaryVsi['healthStatus'],
summary.healthStatus)
self.assertEqual(summaryVsi['totalCapacity'], summary.totalCapacity,
"Total capacity of dimms did not match vsi summary")
self.assertEqual(
summaryVsi['persistentCapacity'], summary.persistentCapacity,
"Persistent capacity of dimms did not match vsi summary")
self.assertEqual(
summaryVsi['availablePersistentCapacity'],
summary.availableCapacity,
"Available persistent capacity of dimms did not match"
" vsi summary")
self.assertEqual(
summaryVsi['numInterleavesets'], summary.numInterleavesets,
"Number of interleave sets did not match vsi summary")
self.assertEqual(summaryVsi['numNamespaces'], summary.numNamespaces,
"Number of namespaces did not match vsi summary")
def buildList():
"""Build list of ssds
"""
ssds = {}
node = "/vmkModules/plog/devices/"
for disk in vsi.list(node):
isSSD = vsi.get(node + disk + "/info")["isSSD"]
if isSSD:
elevNode = node + disk + "/elevStats"
ssds[disk] = {
"state": "QUERY",
"stateval": 0,
"prevval": vsi.get(elevNode)["plogDataUsage"],
"node": elevNode,
}
return ssds
def checkDrainState(ssds):
"""Update list of ssds
"""
done = True
for (diskName, diskInfo) in list(ssds.items()):
_logger.info(diskInfo)
if diskInfo["state"] == "QUERY":
done = False
fullData = vsi.get(diskInfo["node"])
newVal = fullData["plogDataUsage"]
prevVal = diskInfo["prevval"]
_logger.info("Prev Value: %d, New Value: %d", newVal, prevVal)
rate = prevVal - newVal
if rate >= 0:
rate /= _INTERVAL
_logger.info(
"Disk %s plog data drain rate %d MB",
diskName,
rate / (1024 * 1024),
)
elif newVal >= 10000000000000000000:
pass
else:
_logger.error(
"Disk %s plog data increasing %d MB to %d MB",
diskName,
prevVal >> 20,
newVal >> 20,
)
if rate <= -10 * 1024 * 1024: # Too much new data. raise
raise Exception("Disk %s plog data increasing" % diskName)
_logger.debug(
"Disk %s waiting. Usage %d MB, DrainRate %d MB",
diskName,
newVal / (1024 * 1024),
rate / (1024 * 1024),
)
diskInfo["prevval"] = newVal
if newVal <= _DATA_THRESH or newVal >= 10000000000000000000: # or rate <= _RATE_THRESH:
diskInfo["state"] = "WAIT"
diskInfo["stateval"] = 30 # 30 seconds in wait state
_logger.debug("Disk %s drain end condition met", diskName)
continue
if diskInfo["state"] == "WAIT":
diskInfo["stateval"] -= _INTERVAL
if diskInfo["stateval"] <= 0:
diskInfo["state"] = "DONE"
else:
done = False
return done
def TestInterleavesetInfo(self, setinfo):
"""
Test NVDIMM Interleaveset information
"""
global vsi
self.assertIsNotNone(
setinfo, "NVDIMM Interleave set properties should not be None")
# Loop lists to check if interleave set property contents are the same
for s in setinfo:
# Get NVDIMM interleave set properties through VSI call
setidstr = "{0:#0{1}x}".format(s.setId, 10)
v = vsi.get('/hardware/nvd/interleaveset/%s/properties' % setidstr)
self.assertIsNotNone(
v, "NVDIMM interleave set properties obtained via VSI should"
" not be None")
self.assertEqual(
s.setId, v['id'],
"Interleave set ID did not match vsi set property")
self.checkRangeType(v['type'], s.rangeType)
self.assertEqual(
s.baseAddress, v['baseAddress'],
"Interleave set Base Address did not match vsi"
" set property")
self.assertEqual(
s.size, v['size'],
"Interleave set size did not match vsi set property")
self.assertEqual(
s.availableSize, v['availableSize'],
"Interleave set available size did not match vsi"
" set property")
self.checkInterleaveSetState(v['state'], s.state)
dimmList = vsi.list('/hardware/nvd/interleaveset/%s/dimm' %
setidstr)
self.assertIsNotNone(
dimmList,
"List of dimms part of interleave set obtained via VSI should"
" not be None")
for d, l in zip(s.deviceList, dimmList):
# Match device list
dimmstr = "{0:#0{1}x}".format(d, 10)
self.assertEqual(
dimmstr, l, "Interleave set dimm ID did not match vsi"
" device list")
def handleVmciRequests(port):
VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
ECONNABORTED = 103 # Error on non privileged client
bsize = MAX_JSON_SIZE
txt = create_string_buffer(bsize)
cartel = c_int32()
sock = lib.vmci_init(c_uint(port))
if sock == VMCI_ERROR:
errno = get_errno()
raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
% (os.strerror(errno), errno))
skip_count = MAX_SKIP_COUNT # retries for vmci_get_one_op failures
while True:
c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
c, txt.value)
errno = get_errno()
if errno == ECONNABORTED:
logging.warn("Client with non privileged port attempted a request")
continue
if c == VMCI_ERROR:
# We can self-correct by reoping sockets internally. Give it a chance.
logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
c, os.strerror(errno), errno)
skip_count = skip_count - 1
if skip_count <= 0:
raise Exception(
"vmci_get_one_op: too many errors. Giving up.")
continue
else:
skip_count = MAX_SKIP_COUNT # reset the counter, just in case
# Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
str(cartel.value))
group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)
vm_name = group_info["displayName"]
cfg_path = group_info["cfgPath"]
uuid = group_info["uuid"]
# pyVmomi expects uuid like this one: 564dac12-b1a0-f735-0df3-bceb00b30340
# to get it from uuid in VSI vms/<id>/vmmGroup, we use the following format:
UUID_FORMAT = "{0}{1}{2}{3}-{4}{5}-{6}{7}-{8}{9}-{10}{11}{12}{13}{14}{15}"
vm_uuid = UUID_FORMAT.format(*uuid.replace("-", " ").split())
try:
req = json.loads(txt.value.decode('utf-8'))
except ValueError as e:
ret = {u'Error': "Failed to parse json '%s'." % txt.value}
else:
details = req["details"]
opts = details["Opts"] if "Opts" in details else {}
threading.currentThread().setName(vm_name)
ret = executeRequest(vm_uuid=vm_uuid,
vm_name=vm_name,
config_path=cfg_path,
cmd=req["cmd"],
full_vol_name=details["Name"],
opts=opts)
logging.info("executeRequest '%s' completed with ret=%s",
req["cmd"], ret)
ret_string = json.dumps(ret)
response = lib.vmci_reply(c, c_char_p(ret_string.encode()))
errno = get_errno()
logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
if response == VMCI_ERROR:
logging.warning("vmci_reply returned error %s (errno=%d)",
os.strerror(errno), errno)
lib.close(sock) # close listening socket when the loop is over
def dropWrite(wait=None):
"""Drop write cache
wait: wait for dropCache to complete
None => return immediately, DO NOT reset nodes
0 => wait till drop is complete. RESET nodes
other =>
"""
_logger.debug("Dropping write cache")
oldThresh = vsi.get("/vmkModules/plog/plogLogThreshold")
defRunElev = vsi.get("/config/LSOM/intOpts/plogRunElevator")["cur"]
oldSleep = vsi.get("/config/LSOM/intOpts/plogElevSleepTime")["cur"]
if oldThresh < 1000:
_logger.error("Skipping write cache as log thresh is very small %d",
oldThresh)
return
_logger.warning(
"Reset using the following commands after drop cache completes")
_logger.warning("vsish -e set /config/LSOM/intOpts/plogRunElevator %d",
defRunElev)
_logger.warning("vsish -e set /config/LSOM/intOpts/plogElevSleepTime %d",
oldSleep)
defDdpThrottle = None
try:
defDdpThrottle = vsi.get(
"/config/LSOM/intOpts/dedupElevThrottle")["cur"]
_logger.warning(
"vsish -e set /config/LSOM/intOpts/dedupElevThrottle %d",
defDdpThrottle,
)
vsi.set("/config/LSOM/intOpts/dedupElevThrottle", 0)
except:
_logger.warning(
"vsi /config/LSOM/intOpts/dedupElevThrottle does not exist for this build"
)
pass
vsi.set("/config/LSOM/intOpts/plogRunElevator", 1)
vsi.set("/config/LSOM/intOpts/plogElevSleepTime", 1)
if wait is None:
_logger.warning("Returning without resetting vsi nodes")
return
start = time.time()
end = 0
if wait > 0:
end = start + wait
try:
ssds = buildList()
while True:
attempts = 0
while attempts < 10:
try:
done = checkDrainState(ssds)
break
except:
attempts = attempts + 1
_logger.warning("dropcache increase for %r times" %
attempts)
time.sleep(60)
if done:
_logger.warning("Drop cache completed")
break
if end and time.time() > end:
_logger.warning("Drop cache timed out (%d s). Resetting nodes",
wait)
break
time.sleep(_INTERVAL)
finally:
_logger.warning(
"dropWrite returning in %d s, Resetting vsi nodes",
time.time() - start,
)
vsi.set("/config/LSOM/intOpts/plogRunElevator", defRunElev)
vsi.set("/config/LSOM/intOpts/plogElevSleepTime", oldSleep)
try:
vsi.set("/config/LSOM/intOpts/dedupElevThrottle", defDdpThrottle)
except:
_logger.warning(
"vsi /config/LSOM/intOpts/dedupElevThrottle does not exist for this build"
)
pass
def handleVmciRequests(port):
VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
ECONNABORTED = 103 # Error on non privileged client
bsize = MAX_JSON_SIZE
txt = create_string_buffer(bsize)
cartel = c_int32()
sock = lib.vmci_init(c_uint(port))
if sock == VMCI_ERROR:
errno = get_errno()
raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
% (os.strerror(errno), errno))
skip_count = MAX_SKIP_COUNT # retries for vmci_get_one_op failures
while True:
c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
c, txt.value)
errno = get_errno()
if errno == ECONNABORTED:
logging.warn("Client with non privileged port attempted a request")
continue
if c == VMCI_ERROR:
# We can self-correct by reoping sockets internally. Give it a chance.
logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
c, os.strerror(errno), errno)
skip_count = skip_count - 1
if skip_count <= 0:
raise Exception(
"vmci_get_one_op: too many errors. Giving up.")
continue
else:
skip_count = MAX_SKIP_COUNT # reset the counter, just in case
# Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
str(cartel.value))
group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)
vm_name = group_info["displayName"]
cfg_path = group_info["cfgPath"]
uuid = group_info["uuid"]
# pyVmomi expects uuid like this one: 564dac12-b1a0-f735-0df3-bceb00b30340
# to get it from uuid in VSI vms/<id>/vmmGroup, we use the following format:
UUID_FORMAT = "{0}{1}{2}{3}-{4}{5}-{6}{7}-{8}{9}-{10}{11}{12}{13}{14}{15}"
vm_uuid = UUID_FORMAT.format(*uuid.replace("-", " ").split())
try:
req = json.loads(txt.value.decode('utf-8'))
except ValueError as e:
ret = {u'Error': "Failed to parse json '%s'." % txt.value}
else:
details = req["details"]
opts = details["Opts"] if "Opts" in details else {}
threading.currentThread().setName(vm_name)
ret = executeRequest(vm_uuid=vm_uuid,
vm_name=vm_name,
config_path=cfg_path,
cmd=req["cmd"],
full_vol_name=details["Name"],
opts=opts)
logging.info("executeRequest '%s' completed with ret=%s",
req["cmd"], ret)
ret_string = json.dumps(ret)
response = lib.vmci_reply(c, c_char_p(ret_string.encode()))
errno = get_errno()
logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
if response == VMCI_ERROR:
logging.warning("vmci_reply returned error %s (errno=%d)",
os.strerror(errno), errno)
lib.close(sock) # close listening socket when the loop is over
import vmware.vsi as vsi
import time
from collections import defaultdict
def calculate(xyz,nics):
if xyz[0] == 0 and xyz[1] == 0:
print "Lets wait for packets to come to %s" %(nics)
else:
print "The number of packets received for %s in the last 10 seconds are " %(nics.upper())+ str(xyz[2]-xyz[0])
print "The number of packets sent from %s in the last 10 seconds are " %(nics.upper())+ str(xyz[3]-xyz[1])
vmnics = defaultdict(list)
no_nics=vsi.list('/net/pNics/')
print "There are total of %d network cards in the machine " %(len(no_nics))
for i in no_nics:
vmnics[i]=[0,0]
while True:
for nics in no_nics:
ReceivePackets=vsi.get('/net/pNics/%s/stats' % nics)['rxpkt']
vmnics[nics].append(ReceivePackets)
SendPackets=vsi.get('/net/pNics/%s/stats' % nics)['txpkt']
vmnics[nics].append(SendPackets)
calculate(vmnics[nics],nics)
del vmnics[nics][0:2]
print "-"*20 + "Sleeping for 10 seconds" + "-"*20
print 2*'\n'
time.sleep(10)
import time
from collections import defaultdict
def calculate(xyz, nics):
if xyz[0] == 0 and xyz[1] == 0:
print "Lets wait for packets to come to %s" % (nics)
else:
print "The number of packets received for %s in the last 10 seconds are " % (
nics.upper()) + str(xyz[2] - xyz[0])
print "The number of packets sent from %s in the last 10 seconds are " % (
nics.upper()) + str(xyz[3] - xyz[1])
vmnics = defaultdict(list)
no_nics = vsi.list('/net/pNics/')
print "There are total of %d network cards in the machine " % (len(no_nics))
for i in no_nics:
vmnics[i] = [0, 0]
while True:
for nics in no_nics:
ReceivePackets = vsi.get('/net/pNics/%s/stats' % nics)['rxpkt']
vmnics[nics].append(ReceivePackets)
SendPackets = vsi.get('/net/pNics/%s/stats' % nics)['txpkt']
vmnics[nics].append(SendPackets)
calculate(vmnics[nics], nics)
del vmnics[nics][0:2]
print "-" * 20 + "Sleeping for 10 seconds" + "-" * 20
print 2 * '\n'
time.sleep(10)
def TestDimmInfo(self, dimminfo):
"""
Test NVDIMM Information
"""
global vsi
self.assertIsNotNone(dimminfo, "NVDIMM Information should not be None")
# Loop lists to check if dimminfo contents are the same
for d in dimminfo:
# Get NVDIMM Information through VSI call
dimmstr = "{0:#0{1}x}".format(d.dimmHandle, 10)
v = vsi.get('/hardware/nvd/dimm/%s/dimminfo' % dimmstr)
self.assertIsNotNone(
v, "NVDIMM information obtained via VSI should"
" not be None")
self.assertEqual(d.dimmHandle, v['dimmHandle'],
"Dimm Handle did not match vsi dimm handle")
self.assertEqual(
d.totalCapacity, v['totalCapacity'],
"Total Capacity did not match vsi dimm information")
self.assertEqual(
d.persistentCapacity, v['persistentCapacity'],
"Persistent Capacity did not match vsi"
" dimm information")
self.assertEqual(
d.availablePersistentCapacity,
v['availablePersistentCapacity'],
"Available Persistent Capacity did not match vsi"
" dimm information")
self.assertEqual(
d.volatileCapacity, v['volatileCapacity'],
"Volatile Capacity did not match vsi dimm"
" information")
self.assertEqual(
d.availableVolatileCapacity, v['availableVolatileCapacity'],
"Available Volatile Capacity did not match vsi"
" dimm information")
# Get list of regions part of DIMM
regionList = vsi.list('/hardware/nvd/dimm/%s/region' % dimmstr)
self.assertIsNotNone(regionList, "Region list cannot be none")
stateFlags = 0
"""
Get vsi region information and match with hostd obtained
region information.
"""
for vsiregion, r in zip(regionList, d.regionInfo):
regionInfo = vsi.get(
'/hardware/nvd/dimm/%s/region/%s/regioninfo' %
(dimmstr, vsiregion))
self.assertIsNotNone(regionInfo,
"Region information cannot be none")
self.assertEqual(r.regionId, regionInfo['id'],
"Region ID does not match")
self.assertEqual(r.setId, regionInfo['setId'],
"Setid does not match vsi information")
self.checkRangeType(regionInfo['type'], r.rangeType)
self.assertEqual(
r.startAddr, regionInfo['startAddr'],
"Region start address does not match vsi information")
self.assertEqual(r.size, regionInfo['size'],
"Region Size does not match vsi information")
self.assertEqual(
r.offset, regionInfo['offset'],
"Region offset does not match vsi information")
stateFlags = stateFlags | regionInfo['stateFlags']
## Get NVDIMM topology
vsiTopo = vsi.get('/hardware/nvd/dimm/%s/topology' % dimmstr)
## Test NVDIMM representation string
self.assertIsNotNone(vsiTopo, "Topology can not be null")
self.assertEqual(
vsiTopo['representationStr'], d.representationString,
"Representation string does not match vsi information")
## Check health information for DIMM
vsiHealth = vsi.get('/hardware/nvd/dimm/%s/healthinfo' % dimmstr)
self.assertIsNotNone(vsiHealth,
"Health information cannot be none")
self.TestDimmHealthinfo(stateFlags, d.healthInfo, vsiHealth)
def handleVmciRequests():
VMCI_ERROR = -1 # VMCI C code uses '-1' to indicate failures
# Load and use DLL with vsocket shim to listen for docker requests
lib = CDLL(os.path.join(LIB_LOC, "libvmci_srv.so"), use_errno=True)
bsize = MAX_JSON_SIZE
txt = create_string_buffer(bsize)
cartel = c_int32()
sock = lib.vmci_init()
if sock == VMCI_ERROR:
errno = get_errno()
raise OSError("Failed to initialize vSocket listener: %s (errno=%d)" \
% (os.strerror(errno), errno))
skip_count = MAX_SKIP_COUNT # retries for vmci_get_one_op failures
while True:
c = lib.vmci_get_one_op(sock, byref(cartel), txt, c_int(bsize))
logging.debug("lib.vmci_get_one_op returns %d, buffer '%s'",
c, txt.value)
if c == VMCI_ERROR:
# We can self-correct by reoping sockets internally. Give it a chance.
errno = get_errno()
logging.warning("vmci_get_one_op failed ret=%d: %s (errno=%d) Retrying...",
c, os.strerror(errno), errno)
skip_count = skip_count - 1
if skip_count <= 0:
raise Exception(
"vmci_get_one_op: too many errors. Giving up.")
continue
else:
skip_count = MAX_SKIP_COUNT # reset the counter, just in case
# Get VM name & ID from VSI (we only get cartelID from vmci, need to convert)
vmm_leader = vsi.get("/userworld/cartel/%s/vmmLeader" %
str(cartel.value))
group_info = vsi.get("/vm/%s/vmmGroupInfo" % vmm_leader)
vm_name = group_info["displayName"]
cfg_path = group_info["cfgPath"]
try:
req = json.loads(txt.value, "utf-8")
except ValueError as e:
ret = {u'Error': "Failed to parse json '%s'." % txt.value}
else:
details = req["details"]
opts = details["Opts"] if "Opts" in details else {}
ret = executeRequest(vm_name, cfg_path, req["cmd"],
details["Name"], opts)
logging.info("executeRequest '%s' completed with ret=%s",
req["cmd"], ret)
response = lib.vmci_reply(c, c_char_p(json.dumps(ret)))
errno = get_errno()
logging.debug("lib.vmci_reply: VMCI replied with errcode %s", response)
if response == VMCI_ERROR:
logging.warning("vmci_reply returned error %s (errno=%d)",
os.strerror(errno), errno)
lib.close(sock) # close listening socket when the loop is over
