def testUseProcessStats(self): from mars import resource cpu_total = resource.cpu_count() mem_total = resource.virtual_memory().total try: os.environ['MARS_USE_PROCESS_STAT'] = '1' os.environ['MARS_CPU_TOTAL'] = str(cpu_total) os.environ['MARS_MEMORY_TOTAL'] = str(mem_total) resource = importlib.reload(resource) resource.cpu_percent() time.sleep(0.5) mem_stats = resource.virtual_memory() self.assertGreaterEqual(mem_stats.available, 0) self.assertGreaterEqual(mem_stats.total, 0) self.assertGreaterEqual(mem_stats.percent, 0) self.assertGreaterEqual(mem_stats.used, 0) self.assertGreaterEqual(mem_stats.free, 0) cpu_usage = resource.cpu_percent() self.assertGreaterEqual(cpu_usage, 0) cpu_usage = resource.cpu_percent() self.assertGreaterEqual(cpu_usage, 0) finally: del os.environ['MARS_USE_PROCESS_STAT'] del os.environ['MARS_CPU_TOTAL'] del os.environ['MARS_MEMORY_TOTAL'] importlib.reload(resource)
def test_use_process_stats(): from mars import resource cpu_total = resource.cpu_count() mem_total = resource.virtual_memory().total try: os.environ["MARS_USE_PROCESS_STAT"] = "1" os.environ["MARS_CPU_TOTAL"] = str(cpu_total) os.environ["MARS_MEMORY_TOTAL"] = str(mem_total) resource = importlib.reload(resource) resource.cpu_percent() time.sleep(0.5) mem_stats = resource.virtual_memory() assert mem_stats.available >= 0 assert mem_stats.total >= 0 assert mem_stats.percent >= 0 assert mem_stats.used >= 0 assert mem_stats.free >= 0 cpu_usage = resource.cpu_percent() assert cpu_usage >= 0 cpu_usage = resource.cpu_percent() assert cpu_usage >= 0 finally: del os.environ["MARS_USE_PROCESS_STAT"] del os.environ["MARS_CPU_TOTAL"] del os.environ["MARS_MEMORY_TOTAL"] importlib.reload(resource)
def testStats(self): from mars import resource resource = importlib.reload(resource) resource.cpu_percent() mem_stats = resource.virtual_memory() self.assertGreaterEqual(mem_stats.available, 0) self.assertGreaterEqual(mem_stats.total, 0) self.assertGreaterEqual(mem_stats.percent, 0) self.assertGreaterEqual(mem_stats.used, 0) self.assertGreaterEqual(mem_stats.free, 0) cpu_usage = resource.cpu_percent() time.sleep(0.1) self.assertGreaterEqual(cpu_usage, 0) resource.disk_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.disk_io_usage() self.assertGreaterEqual(recv_speed, 0) self.assertGreaterEqual(send_speed, 0) curdir = os.path.dirname(os.path.abspath(__file__)) resource.disk_io_usage(curdir) time.sleep(0.1) usage = resource.disk_io_usage(curdir) if usage is not None: self.assertGreaterEqual(usage.reads, 0) self.assertGreaterEqual(usage.writes, 0) resource.net_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.net_io_usage() self.assertGreaterEqual(recv_speed, 0) self.assertGreaterEqual(send_speed, 0)
def testStats(self): from mars import resource resource = reload_module(resource) resource.cpu_percent() mem_stats = resource.virtual_memory() self.assertGreaterEqual(mem_stats.available, 0) self.assertGreaterEqual(mem_stats.total, 0) self.assertGreaterEqual(mem_stats.percent, 0) self.assertGreaterEqual(mem_stats.used, 0) self.assertGreaterEqual(mem_stats.free, 0) cpu_usage = resource.cpu_percent() time.sleep(0.1) self.assertGreaterEqual(cpu_usage, 0) resource.disk_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.disk_io_usage() self.assertGreaterEqual(recv_speed, 0) self.assertGreaterEqual(send_speed, 0) resource.net_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.net_io_usage() self.assertGreaterEqual(recv_speed, 0) self.assertGreaterEqual(send_speed, 0)
def test_stats(): from mars import resource resource = importlib.reload(resource) resource.cpu_percent() mem_stats = resource.virtual_memory() assert mem_stats.available >= 0 assert mem_stats.total >= 0 assert mem_stats.percent >= 0 assert mem_stats.used >= 0 assert mem_stats.free >= 0 cpu_usage = resource.cpu_percent() time.sleep(0.1) assert cpu_usage >= 0 resource.disk_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.disk_io_usage() assert recv_speed >= 0 assert send_speed >= 0 curdir = os.path.dirname(os.path.abspath(__file__)) resource.disk_io_usage(curdir) time.sleep(0.1) usage = resource.disk_io_usage(curdir) if usage is not None: assert usage.reads >= 0 assert usage.writes >= 0 resource.net_io_usage() time.sleep(0.1) recv_speed, send_speed = resource.net_io_usage() assert recv_speed >= 0 assert send_speed >= 0
def test_use_c_group_stats(): from mars import resource fd, cpu_stat_path = tempfile.mkstemp(prefix="test-mars-res-cpu-") with os.fdopen(fd, "w") as f: f.write(_cpu_stat_first) fd, mem_stat_path = tempfile.mkstemp(prefix="test-mars-res-mem-") with os.fdopen(fd, "w") as f: f.write(_memory_stat_content) old_cpu_stat_file = resource.CGROUP_CPU_STAT_FILE old_mem_stat_file = resource.CGROUP_MEM_STAT_FILE old_shm_path = resource._shm_path try: os.environ["MARS_USE_CGROUP_STAT"] = "1" resource = importlib.reload(resource) resource.CGROUP_CPU_STAT_FILE = cpu_stat_path resource.CGROUP_MEM_STAT_FILE = mem_stat_path resource._shm_path = None assert resource.cpu_percent() is None time.sleep(0.5) with open(cpu_stat_path, "w") as f: f.write(_cpu_stat_last) assert resource.cpu_percent() > 50 assert resource.cpu_percent() < 150 mem_stats = resource.virtual_memory() assert mem_stats.total == 1073741824 assert mem_stats.used == 218181632 finally: resource.CGROUP_CPU_STAT_FILE = old_cpu_stat_file resource.CGROUP_MEM_STAT_FILE = old_mem_stat_file resource._shm_path = old_shm_path del os.environ["MARS_USE_CGROUP_STAT"] os.unlink(cpu_stat_path) os.unlink(mem_stat_path) importlib.reload(resource)
def testUseCGroupStats(self): from mars import resource fd, cpu_stat_path = tempfile.mkstemp(prefix='test-mars-res-cpu-') with os.fdopen(fd, 'w') as f: f.write(_cpu_stat_first) fd, mem_stat_path = tempfile.mkstemp(prefix='test-mars-res-mem-') with os.fdopen(fd, 'w') as f: f.write(_memory_stat_content) old_cpu_stat_file = resource.CGROUP_CPU_STAT_FILE old_mem_stat_file = resource.CGROUP_MEM_STAT_FILE old_shm_path = resource._shm_path try: os.environ['MARS_USE_CGROUP_STAT'] = '1' resource = importlib.reload(resource) resource.CGROUP_CPU_STAT_FILE = cpu_stat_path resource.CGROUP_MEM_STAT_FILE = mem_stat_path resource._shm_path = None self.assertIsNone(resource.cpu_percent()) time.sleep(0.5) with open(cpu_stat_path, 'w') as f: f.write(_cpu_stat_last) self.assertGreater(resource.cpu_percent(), 50) self.assertLess(resource.cpu_percent(), 150) mem_stats = resource.virtual_memory() self.assertEqual(mem_stats.total, 1073741824) self.assertEqual(mem_stats.used, 218181632) finally: resource.CGROUP_CPU_STAT_FILE = old_cpu_stat_file resource.CGROUP_MEM_STAT_FILE = old_mem_stat_file resource._shm_path = old_shm_path del os.environ['MARS_USE_CGROUP_STAT'] os.unlink(cpu_stat_path) os.unlink(mem_stat_path) importlib.reload(resource)
def gather_node_info(): from .lib.mkl_interface import mkl_get_version mem_stats = resource.virtual_memory() node_info = dict() node_info['command_line'] = ' '.join(sys.argv) node_info['platform'] = platform.platform() node_info['host_name'] = socket.gethostname() node_info['sys_version'] = sys.version node_info['cpu_info'] = 'Used: %f\nTotal: %d' % ( resource.cpu_percent() / 100.0, resource.cpu_count()) node_info['memory_info'] = 'Used: %s\nTotal: %s' % (readable_size( mem_stats.used), readable_size(mem_stats.total)) for collector in _collectors.values(): node_info.update(collector()) if np is None: node_info['numpy_info'] = 'Not installed' else: sio = six.StringIO() sio.write('Version: %s\n' % np.__version__) if hasattr(np, '__mkl_version__') and mkl_get_version: mkl_version = mkl_get_version() sio.write( 'MKL Version: %d.%d.%d\n' % (mkl_version.major, mkl_version.minor, mkl_version.update)) node_info['numpy_info'] = sio.getvalue().strip() if scipy is None: node_info['scipy_info'] = 'Not installed' else: node_info['scipy_info'] = 'Version: %s' % scipy.__version__ git = git_info() if git: node_info['git_info'] = '%s %s' % (git[0], git[1]) else: node_info['git_info'] = 'Not available' return node_info