def test_use_process_stats():
from mars import resource
cpu_total = resource.cpu_count()
mem_total = resource.virtual_memory().total
try:
os.environ["MARS_USE_PROCESS_STAT"] = "1"
os.environ["MARS_CPU_TOTAL"] = str(cpu_total)
os.environ["MARS_MEMORY_TOTAL"] = str(mem_total)
resource = importlib.reload(resource)
resource.cpu_percent()
time.sleep(0.5)
mem_stats = resource.virtual_memory()
assert mem_stats.available >= 0
assert mem_stats.total >= 0
assert mem_stats.percent >= 0
assert mem_stats.used >= 0
assert mem_stats.free >= 0
cpu_usage = resource.cpu_percent()
assert cpu_usage >= 0
cpu_usage = resource.cpu_percent()
assert cpu_usage >= 0
finally:
del os.environ["MARS_USE_PROCESS_STAT"]
del os.environ["MARS_CPU_TOTAL"]
del os.environ["MARS_MEMORY_TOTAL"]
importlib.reload(resource)
def testUseProcessStats(self):
from mars import resource
cpu_total = resource.cpu_count()
mem_total = resource.virtual_memory().total
try:
os.environ['MARS_USE_PROCESS_STAT'] = '1'
os.environ['MARS_CPU_TOTAL'] = str(cpu_total)
os.environ['MARS_MEMORY_TOTAL'] = str(mem_total)
resource = importlib.reload(resource)
resource.cpu_percent()
time.sleep(0.5)
mem_stats = resource.virtual_memory()
self.assertGreaterEqual(mem_stats.available, 0)
self.assertGreaterEqual(mem_stats.total, 0)
self.assertGreaterEqual(mem_stats.percent, 0)
self.assertGreaterEqual(mem_stats.used, 0)
self.assertGreaterEqual(mem_stats.free, 0)
cpu_usage = resource.cpu_percent()
self.assertGreaterEqual(cpu_usage, 0)
cpu_usage = resource.cpu_percent()
self.assertGreaterEqual(cpu_usage, 0)
finally:
del os.environ['MARS_USE_PROCESS_STAT']
del os.environ['MARS_CPU_TOTAL']
del os.environ['MARS_MEMORY_TOTAL']
importlib.reload(resource)
def testStats(self):
from mars import resource
resource = importlib.reload(resource)
resource.cpu_percent()
mem_stats = resource.virtual_memory()
self.assertGreaterEqual(mem_stats.available, 0)
self.assertGreaterEqual(mem_stats.total, 0)
self.assertGreaterEqual(mem_stats.percent, 0)
self.assertGreaterEqual(mem_stats.used, 0)
self.assertGreaterEqual(mem_stats.free, 0)
cpu_usage = resource.cpu_percent()
time.sleep(0.1)
self.assertGreaterEqual(cpu_usage, 0)
resource.disk_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.disk_io_usage()
self.assertGreaterEqual(recv_speed, 0)
self.assertGreaterEqual(send_speed, 0)
curdir = os.path.dirname(os.path.abspath(__file__))
resource.disk_io_usage(curdir)
time.sleep(0.1)
usage = resource.disk_io_usage(curdir)
if usage is not None:
self.assertGreaterEqual(usage.reads, 0)
self.assertGreaterEqual(usage.writes, 0)
resource.net_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.net_io_usage()
self.assertGreaterEqual(recv_speed, 0)
self.assertGreaterEqual(send_speed, 0)
def testStats(self):
from mars import resource
resource = reload_module(resource)
resource.cpu_percent()
mem_stats = resource.virtual_memory()
self.assertGreaterEqual(mem_stats.available, 0)
self.assertGreaterEqual(mem_stats.total, 0)
self.assertGreaterEqual(mem_stats.percent, 0)
self.assertGreaterEqual(mem_stats.used, 0)
self.assertGreaterEqual(mem_stats.free, 0)
cpu_usage = resource.cpu_percent()
time.sleep(0.1)
self.assertGreaterEqual(cpu_usage, 0)
resource.disk_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.disk_io_usage()
self.assertGreaterEqual(recv_speed, 0)
self.assertGreaterEqual(send_speed, 0)
resource.net_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.net_io_usage()
self.assertGreaterEqual(recv_speed, 0)
self.assertGreaterEqual(send_speed, 0)
def test_stats():
from mars import resource
resource = importlib.reload(resource)
resource.cpu_percent()
mem_stats = resource.virtual_memory()
assert mem_stats.available >= 0
assert mem_stats.total >= 0
assert mem_stats.percent >= 0
assert mem_stats.used >= 0
assert mem_stats.free >= 0
cpu_usage = resource.cpu_percent()
time.sleep(0.1)
assert cpu_usage >= 0
resource.disk_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.disk_io_usage()
assert recv_speed >= 0
assert send_speed >= 0
curdir = os.path.dirname(os.path.abspath(__file__))
resource.disk_io_usage(curdir)
time.sleep(0.1)
usage = resource.disk_io_usage(curdir)
if usage is not None:
assert usage.reads >= 0
assert usage.writes >= 0
resource.net_io_usage()
time.sleep(0.1)
recv_speed, send_speed = resource.net_io_usage()
assert recv_speed >= 0
assert send_speed >= 0
def testUseCGroupStats(self):
from mars import resource
fd, mem_stat_path = tempfile.mkstemp(prefix='test-mars-res-')
with os.fdopen(fd, 'w') as f:
f.write(_memory_stat_content)
old_stat_file = resource.CGROUP_MEM_STAT_FILE
try:
os.environ['MARS_MEM_USE_CGROUP_STAT'] = '1'
resource = reload_module(resource)
resource.CGROUP_MEM_STAT_FILE = mem_stat_path
mem_stats = resource.virtual_memory()
self.assertEqual(mem_stats.total, 1073741824)
self.assertEqual(mem_stats.used, 707457024)
finally:
resource.CGROUP_MEM_STAT_FILE = old_stat_file
del os.environ['MARS_MEM_USE_CGROUP_STAT']
os.unlink(mem_stat_path)
reload_module(resource)
def gather_node_info():
from .lib.mkl_interface import mkl_get_version
mem_stats = resource.virtual_memory()
node_info = dict()
node_info['command_line'] = ' '.join(sys.argv)
node_info['platform'] = platform.platform()
node_info['host_name'] = socket.gethostname()
node_info['sys_version'] = sys.version
node_info['cpu_info'] = 'Used: %f\nTotal: %d' % (
resource.cpu_percent() / 100.0, resource.cpu_count())
node_info['memory_info'] = 'Used: %s\nTotal: %s' % (readable_size(
mem_stats.used), readable_size(mem_stats.total))
for collector in _collectors.values():
node_info.update(collector())
if np is None:
node_info['numpy_info'] = 'Not installed'
else:
sio = six.StringIO()
sio.write('Version: %s\n' % np.__version__)
if hasattr(np, '__mkl_version__') and mkl_get_version:
mkl_version = mkl_get_version()
sio.write(
'MKL Version: %d.%d.%d\n' %
(mkl_version.major, mkl_version.minor, mkl_version.update))
node_info['numpy_info'] = sio.getvalue().strip()
if scipy is None:
node_info['scipy_info'] = 'Not installed'
else:
node_info['scipy_info'] = 'Version: %s' % scipy.__version__
git = git_info()
if git:
node_info['git_info'] = '%s %s' % (git[0], git[1])
else:
node_info['git_info'] = 'Not available'
return node_info
def test_use_c_group_stats():
from mars import resource
fd, cpu_stat_path = tempfile.mkstemp(prefix="test-mars-res-cpu-")
with os.fdopen(fd, "w") as f:
f.write(_cpu_stat_first)
fd, mem_stat_path = tempfile.mkstemp(prefix="test-mars-res-mem-")
with os.fdopen(fd, "w") as f:
f.write(_memory_stat_content)
old_cpu_stat_file = resource.CGROUP_CPU_STAT_FILE
old_mem_stat_file = resource.CGROUP_MEM_STAT_FILE
old_shm_path = resource._shm_path
try:
os.environ["MARS_USE_CGROUP_STAT"] = "1"
resource = importlib.reload(resource)
resource.CGROUP_CPU_STAT_FILE = cpu_stat_path
resource.CGROUP_MEM_STAT_FILE = mem_stat_path
resource._shm_path = None
assert resource.cpu_percent() is None
time.sleep(0.5)
with open(cpu_stat_path, "w") as f:
f.write(_cpu_stat_last)
assert resource.cpu_percent() > 50
assert resource.cpu_percent() < 150
mem_stats = resource.virtual_memory()
assert mem_stats.total == 1073741824
assert mem_stats.used == 218181632
finally:
resource.CGROUP_CPU_STAT_FILE = old_cpu_stat_file
resource.CGROUP_MEM_STAT_FILE = old_mem_stat_file
resource._shm_path = old_shm_path
del os.environ["MARS_USE_CGROUP_STAT"]
os.unlink(cpu_stat_path)
os.unlink(mem_stat_path)
importlib.reload(resource)
def testUseCGroupStats(self):
from mars import resource
fd, cpu_stat_path = tempfile.mkstemp(prefix='test-mars-res-cpu-')
with os.fdopen(fd, 'w') as f:
f.write(_cpu_stat_first)
fd, mem_stat_path = tempfile.mkstemp(prefix='test-mars-res-mem-')
with os.fdopen(fd, 'w') as f:
f.write(_memory_stat_content)
old_cpu_stat_file = resource.CGROUP_CPU_STAT_FILE
old_mem_stat_file = resource.CGROUP_MEM_STAT_FILE
old_shm_path = resource._shm_path
try:
os.environ['MARS_USE_CGROUP_STAT'] = '1'
resource = importlib.reload(resource)
resource.CGROUP_CPU_STAT_FILE = cpu_stat_path
resource.CGROUP_MEM_STAT_FILE = mem_stat_path
resource._shm_path = None
self.assertIsNone(resource.cpu_percent())
time.sleep(0.5)
with open(cpu_stat_path, 'w') as f:
f.write(_cpu_stat_last)
self.assertGreater(resource.cpu_percent(), 50)
self.assertLess(resource.cpu_percent(), 150)
mem_stats = resource.virtual_memory()
self.assertEqual(mem_stats.total, 1073741824)
self.assertEqual(mem_stats.used, 218181632)
finally:
resource.CGROUP_CPU_STAT_FILE = old_cpu_stat_file
resource.CGROUP_MEM_STAT_FILE = old_mem_stat_file
resource._shm_path = old_shm_path
del os.environ['MARS_USE_CGROUP_STAT']
os.unlink(cpu_stat_path)
os.unlink(mem_stat_path)
importlib.reload(resource)
