def crawl(self, container_id, avoid_setns=False, **kwargs): container = DockerContainer(container_id) used = buffered = cached = free = 'unknown' with open(container.get_memory_cgroup_path('memory.stat'), 'r') as f: for line in f: (key, value) = line.strip().split(' ') if key == 'total_cache': cached = int(value) if key == 'total_active_file': buffered = int(value) with open(container.get_memory_cgroup_path('memory.limit_in_bytes'), 'r') as f: limit = int(f.readline().strip()) with open(container.get_memory_cgroup_path('memory.usage_in_bytes'), 'r') as f: used = int(f.readline().strip()) host_free = psutil.virtual_memory().free container_total = used + min(host_free, limit - used) free = container_total - used if 'unknown' not in [used, free] and (free + used) > 0: util_percentage = float(used) / (free + used) * 100.0 else: util_percentage = 'unknown' return [('memory', MemoryFeature(used, buffered, cached, free, util_percentage), 'memory')]
def crawl(self, container_id, avoid_setns=False, **kwargs): container = DockerContainer(container_id) used = buffered = cached = free = 'unknown' with open(container.get_memory_cgroup_path('memory.stat' ), 'r') as f: for line in f: (key, value) = line.strip().split(' ') if key == 'total_cache': cached = int(value) if key == 'total_active_file': buffered = int(value) with open(container.get_memory_cgroup_path( 'memory.limit_in_bytes'), 'r') as f: limit = int(f.readline().strip()) with open(container.get_memory_cgroup_path( 'memory.usage_in_bytes'), 'r') as f: used = int(f.readline().strip()) host_free = psutil.virtual_memory().free container_total = used + min(host_free, limit - used) free = container_total - used if 'unknown' not in [used, free] and (free + used) > 0: util_percentage = float(used) / (free + used) * 100.0 else: util_percentage = 'unknown' return [('memory', MemoryFeature(used, buffered, cached, free, util_percentage), 'memory')]
def test_link_logfiles(self, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles() mock_symlink.assert_called_with( '/tmp/something/docker/valid_rootfs_id/var/log/2', '/var/log/crawler_container_logs/random_prefix/var/log/2') assert mock_symlink.call_count == 4
def test_is_docker( self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.is_docker_container() print(c)
def test_link_logfiles_symlink_exception( self, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles()
def test_init_wrong_environment(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): with self.assertRaises(crawler_exceptions.ContainerInvalidEnvironment): DockerContainer("no_namespace") with self.assertRaises(crawler_exceptions.ContainerInvalidEnvironment): DockerContainer("throw_bad_environment_exception_id") with self.assertRaises(Exception): DockerContainer("throw_non_handled_exception_id") with self.assertRaises(crawler_exceptions.ContainerInvalidEnvironment): DockerContainer("throw_value_error_id")
def test_memory_cgroup( self, mocked_ismount, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.get_memory_cgroup_path( 'abc') == '/cgroup/memory/docker/good_id/abc'
def _test_non_implemented_methods(self): c = DockerContainer("some_id") with self.assertRaises(NotImplementedError): c.get_memory_cgroup_path() with self.assertRaises(NotImplementedError): c.get_cpu_cgroup_path() with self.assertRaises(NotImplementedError): c.link_logfiles() with self.assertRaises(NotImplementedError): c.unlink_logfiles()
def test_cpu_cgroup( self, mocked_ismount, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.get_cpu_cgroup_path( 'abc') == ("/cgroup/cpuacct/docker/good_id/" "abc") or ("cgroup/cpu,cpuacct/docker/good_id/abc")
def test_link_logfiles( self, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles() mock_symlink.assert_called_with( '/tmp/something/docker/valid_rootfs_id/var/log/2', '/var/log/crawler_container_logs/random_prefix/var/log/2') assert mock_symlink.call_count == 4
def crawl(self, container_id, avoid_setns=False, **kwargs): logger.debug('Crawling %s for container %s' % (self.get_feature(), container_id)) container = DockerContainer(container_id) if avoid_setns: raise NotImplementedError('avoidsetns mode not implemented') else: interfaces = run_as_another_namespace( container.pid, ['net'], self._crawl_interface_counters) for (ifname, curr_count) in interfaces: feature_key = '{0}-{1}'.format('interface', ifname) cache_key = '{0}-{1}-{2}'.format(container.long_id, container.pid, feature_key) (prev_count, prev_time) = self._cache_get_value(cache_key) self._cache_put_value(cache_key, curr_count) if prev_count and prev_time: d = time.time() - prev_time diff = [(a - b) / d for (a, b) in zip(curr_count, prev_count)] else: # first measurement diff = [0] * 6 feature_attributes = InterfaceFeature._make(diff) yield (feature_key, feature_attributes, 'interface')
def test_init_from_inspect_w_repotags2(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): inspect = { "Id": "good_id", "Created": "2016-07-06T16:38:05.479090842Z", "State": { "Status": "running", "Running": True, "Pid": 11186 }, "Image": "sha256:07c86167cdc4264926fa5d2894e34a339ad27", "Name": "/pensive_rosalind", "Mounts": [], "Config": { "Cmd": ["bash"], "Image": "ubuntu:trusty" }, "NetworkSettings": {}, 'RepoTag': 'registry.com:123/img:latest' } c = DockerContainer("good_id", inspect) mock_inspect.assert_not_called() assert not c.root_fs assert mocked_get_runtime_env.call_count == 1 assert c.docker_image_long_name == 'registry.com:123/img:latest' assert c.docker_image_short_name == 'img:latest' assert c.docker_image_tag == 'latest' assert c.docker_image_registry == 'registry.com:123' assert c.owner_namespace == ''
def test_link_and_unlink_docker_json_logfile( self, mock_json_logs, mock_rmtree, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles() mock_symlink.assert_called_with( '/var/lib/docker/abc/container/log.json', '/var/log/crawler_container_logs/random_prefix/docker.log') c.unlink_logfiles() assert mock_symlink.call_count == 5 assert mock_rmtree.call_count == 1
def crawl(self, container_id, avoid_setns=False, **kwargs): container = DockerContainer(container_id) logger.debug('Crawling %s for container %s' % (self.get_feature(), container_id)) if avoid_setns: raise NotImplementedError() else: # in all other cases, including wrong mode set return run_as_another_namespace(container.pid, ALL_NAMESPACES, self.crawl_load)
def test_links_with_volumes(self, mock_rmtree, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): inspect = { "Id": "valid_rootfs_id", "Created": "2016-07-06T16:38:05.479090842Z", "State": { "Status": "running", "Running": True, "Pid": 11186 }, "Image": "sha256:07c86167cdc4264926fa5d2894e34a339ad27", "Name": "/pensive_rosalind", # /var in the container is mapped to /mount/in/the/host # container was started with -v /var/in/the/host:/var "Volumes": { '/var': '/var/in/the/host' }, "Config": { "Cmd": ["bash"], "Image": "ubuntu:trusty" }, "NetworkSettings": {} } c = DockerContainer("valid_rootfs_id", inspect) c.link_logfiles() mock_symlink.assert_called_with( '/var/in/the/host/log/2', '/var/log/crawler_container_logs/random_prefix/var/log/2') c.unlink_logfiles() assert mock_symlink.call_count == 4
def test_links_with_mounts( self, mock_rmtree, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): inspect = { "Id": "valid_rootfs_id", "Created": "2016-07-06T16:38:05.479090842Z", "State": { "Status": "running", "Running": True, "Pid": 11186 }, "Image": "sha256:07c86167cdc4264926fa5d2894e34a339ad27", "Name": "/pensive_rosalind", # /var in the container is mapped to /mount/in/the/host # container was started with -v /var/in/the/host:/var "Mounts": [{'Source': '/var/in/the/host', 'Destination': '/var'}], "Config": { "Cmd": [ "bash" ], "Image": "ubuntu:trusty" }, "NetworkSettings": { } } c = DockerContainer("valid_rootfs_id", inspect) c.link_logfiles() mock_symlink.assert_called_with( '/var/in/the/host/log/2', '/var/log/crawler_container_logs/random_prefix/var/log/2') c.unlink_logfiles() assert mock_symlink.call_count == 4
def test_link_and_unlink_docker_json_logfile(self, mock_json_logs, mock_rmtree, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles() mock_symlink.assert_called_with( '/var/lib/docker/abc/container/log.json', '/var/log/crawler_container_logs/random_prefix/docker.log') c.unlink_logfiles() assert mock_symlink.call_count == 5 assert mock_rmtree.call_count == 1
def __init__(self, max_slots, host=None, ipv6_prefix=None, starting_ipv4_port=None, dockermanager=None): super(DockerMaster, self).__init__(str(uuid.uuid4()), [ 'docker-container', 'docker-container_100M', 'docker-container-with-tunnel' ]) if starting_ipv4_port is None or starting_ipv4_port <= 1024: starting_ipv4_port = 12000 #Default if dockermanager is None: dockermanager = DockerManager() if host is None or len(host) == 0: host = urlopen('http://ip.42.pl/raw').read() self.pool = [ DockerContainer(self, starting_ipv4_port, dockermanager, host, ipv6_prefix) for _ in range(max_slots) ]
def test_init_from_inspect(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): inspect = { "Id": "good_id", "Created": "2016-07-06T16:38:05.479090842Z", "State": { "Status": "running", "Running": True, "Pid": 11186 }, "Image": "sha256:07c86167cdc4264926fa5d2894e34a339ad27", "Name": "/pensive_rosalind", "Mounts": [], "Config": { "Cmd": ["bash"], "Image": "ubuntu:trusty" }, "NetworkSettings": {} } c = DockerContainer("good_id", inspect) mock_inspect.assert_not_called() assert not c.root_fs assert mocked_get_runtime_env.call_count == 1
def _test_is_running(self, mock_exists): c = DockerContainer("good_id") assert c.is_running()
def crawl(self, container_id, avoid_setns=False, per_cpu=False, **kwargs): logger.debug( 'Crawling %s for container %s' % (self.get_feature(), container_id)) container = DockerContainer(container_id) host_cpu_feature = {} for (idx, cpu) in enumerate(psutil.cpu_times_percent(percpu=True)): host_cpu_feature[idx] = CpuFeature( cpu.idle, cpu.nice, cpu.user, cpu.iowait, cpu.system, cpu.irq, cpu.steal, 100 - int(cpu.idle), ) if per_cpu: stat_file_name = 'cpuacct.usage_percpu' else: stat_file_name = 'cpuacct.usage' (cpu_usage_t1, prev_time) = ( self._get_prev_container_cpu_times(container.long_id)) if cpu_usage_t1: logger.debug('Using previous cpu times for container %s' % container.long_id) interval = time.time() - prev_time else: logger.debug( 'There are no previous cpu times for container %s ' 'so we will be sleeping for 100 milliseconds' % container.long_id) with open(container.get_cpu_cgroup_path(stat_file_name), 'r') as f: cpu_usage_t1 = f.readline().strip().split(' ') interval = 0.1 # sleep for 100ms time.sleep(interval) with open(container.get_cpu_cgroup_path(stat_file_name), 'r') as f: cpu_usage_t2 = f.readline().strip().split(' ') # Store the cpu times for the next crawl self._save_container_cpu_times(container.long_id, cpu_usage_t2) cpu_user_system = {} path = container.get_cpu_cgroup_path('cpuacct.stat') with open(path, 'r') as f: for line in f: m = re.search(r"(system|user)\s+(\d+)", line) if m: cpu_user_system[m.group(1)] = \ float(m.group(2)) for (index, cpu_usage_ns) in enumerate(cpu_usage_t1): usage_secs = (float(cpu_usage_t2[index]) - float(cpu_usage_ns)) / float(1e9) # Interval is never 0 because of step 0 (forcing a sleep) usage_percent = usage_secs / interval * 100.0 if usage_percent > 100.0: usage_percent = 100.0 idle = 100.0 - usage_percent # Approximation 1 user_plus_sys_hz = cpu_user_system['user'] \ + cpu_user_system['system'] if user_plus_sys_hz == 0: # Fake value to avoid divide by zero. user_plus_sys_hz = 0.1 user = usage_percent * (cpu_user_system['user'] / user_plus_sys_hz) system = usage_percent * (cpu_user_system['system'] / user_plus_sys_hz) # Approximation 2 nice = host_cpu_feature[index][1] wait = host_cpu_feature[index][3] interrupt = host_cpu_feature[index][5] steal = host_cpu_feature[index][6] feature_key = '{0}-{1}'.format('cpu', index) feature_attributes = CpuFeature( idle, nice, user, wait, system, interrupt, steal, usage_percent, ) yield (feature_key, feature_attributes, 'cpu')
def test_init(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") mock_inspect.assert_called() assert not c.root_fs assert mocked_get_runtime_env.call_count == 1
def test_init_failed(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): with self.assertRaises(crawler_exceptions.ContainerNonExistent): DockerContainer("no_container_id") assert mocked_get_runtime_env.call_count == 0
def test_memory_cgroup(self, mocked_ismount, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.get_memory_cgroup_path( 'abc') == '/cgroup/memory/docker/good_id/abc'
def test_is_docker(self, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.is_docker_container() print(c)
def _test_eq_ne(self): c1 = DockerContainer("good_id") c2 = DockerContainer("ebcd") c3 = DockerContainer("ebcd") assert c1 != c2 assert c2 == c3
def test_cpu_cgroup(self, mocked_ismount, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("good_id") assert c.get_cpu_cgroup_path('abc') == ( "/cgroup/cpuacct/docker/good_id/" "abc") or ("cgroup/cpu,cpuacct/docker/good_id/abc")
def test_link_logfiles_symlink_exception(self, mock_symlink, mock_makedirs, mock_get_rootfs, mock_inspect, mocked_get_runtime_env, mocked_dockerps): c = DockerContainer("valid_rootfs_id") c.link_logfiles()
def crawl(self, container_id, avoid_setns=False, per_cpu=False, **kwargs): logger.debug('Crawling %s for container %s' % (self.get_feature(), container_id)) container = DockerContainer(container_id) host_cpu_feature = {} for (idx, cpu) in enumerate(psutil.cpu_times_percent(percpu=True)): host_cpu_feature[idx] = CpuFeature( cpu.idle, cpu.nice, cpu.user, cpu.iowait, cpu.system, cpu.irq, cpu.steal, 100 - int(cpu.idle), ) if per_cpu: stat_file_name = 'cpuacct.usage_percpu' else: stat_file_name = 'cpuacct.usage' (cpu_usage_t1, prev_time) = (self._get_prev_container_cpu_times(container.long_id)) if cpu_usage_t1: logger.debug('Using previous cpu times for container %s' % container.long_id) interval = time.time() - prev_time else: logger.debug('There are no previous cpu times for container %s ' 'so we will be sleeping for 100 milliseconds' % container.long_id) with open(container.get_cpu_cgroup_path(stat_file_name), 'r') as f: cpu_usage_t1 = f.readline().strip().split(' ') interval = 0.1 # sleep for 100ms time.sleep(interval) with open(container.get_cpu_cgroup_path(stat_file_name), 'r') as f: cpu_usage_t2 = f.readline().strip().split(' ') # Store the cpu times for the next crawl self._save_container_cpu_times(container.long_id, cpu_usage_t2) cpu_user_system = {} path = container.get_cpu_cgroup_path('cpuacct.stat') with open(path, 'r') as f: for line in f: m = re.search(r"(system|user)\s+(\d+)", line) if m: cpu_user_system[m.group(1)] = \ float(m.group(2)) for (index, cpu_usage_ns) in enumerate(cpu_usage_t1): usage_secs = (float(cpu_usage_t2[index]) - float(cpu_usage_ns)) / float(1e9) # Interval is never 0 because of step 0 (forcing a sleep) usage_percent = usage_secs / interval * 100.0 if usage_percent > 100.0: usage_percent = 100.0 idle = 100.0 - usage_percent # Approximation 1 user_plus_sys_hz = cpu_user_system['user'] \ + cpu_user_system['system'] if user_plus_sys_hz == 0: # Fake value to avoid divide by zero. user_plus_sys_hz = 0.1 user = usage_percent * (cpu_user_system['user'] / user_plus_sys_hz) system = usage_percent * (cpu_user_system['system'] / user_plus_sys_hz) # Approximation 2 nice = host_cpu_feature[index][1] wait = host_cpu_feature[index][3] interrupt = host_cpu_feature[index][5] steal = host_cpu_feature[index][6] feature_key = '{0}-{1}'.format('cpu', index) feature_attributes = CpuFeature( idle, nice, user, wait, system, interrupt, steal, usage_percent, ) yield (feature_key, feature_attributes, 'cpu')
def _test_to_str(self): c = DockerContainer("good_id") print(c)